def proc_data(**kwargs):
logger.info(str_kwargs(kwargs, title='Input parameters:'))
DSNAME = kwargs.get(
'dsname',
'/reg/g/psdm/detector/data2_test/xtc/data-amox27716-r0100-acqiris-e000100.xtc2'
)
DETNAME = kwargs.get('detname', 'tmo_quadanode')
EVSKIP = kwargs.get('evskip', 0)
EVENTS = kwargs.get('events', 100) + EVSKIP
OFPREFIX = kwargs.get('ofprefix', './')
VERBOSE = kwargs.get('verbose', False)
ds = DataSource(files=DSNAME)
orun = next(ds.runs())
det = orun.Detector(DETNAME)
kwargs['consts'] = det.calibconst
peaks = WFPeaks(**kwargs)
proc = DLDProcessor(**kwargs)
stats = DLDStatistics(proc, **kwargs)
for nev, evt in enumerate(orun.events()):
if nev < EVSKIP: continue
if nev > EVENTS: break
if do_print(nev): logger.info('Event %3d' % nev)
t0_sec = time()
wts = det.raw.times(evt)
wfs = det.raw.waveforms(evt)
nhits, pkinds, pkvals, pktsec = peaks(wfs, wts)
if VERBOSE:
print(" waveforms processing time = %.6f sec" % (time() - t0_sec))
print_ndarr(wfs, ' waveforms : ', last=4)
print_ndarr(wts, ' times : ', last=4)
print_ndarr(nhits, ' number_of_hits: ')
print_ndarr(pktsec, ' peak_times_sec: ', last=4)
proc.event_proc(nev, nhits, pktsec)
stats.fill_data(nhits, pktsec)
if VERBOSE:
for i, (x, y, r,
t) in enumerate(proc.xyrt_list(nev, nhits, pktsec)):
print(' hit:%2d x:%7.3f y:%7.3f t:%10.5g r:%7.3f' %
(i, x, y, t, r))
draw_plots(stats, prefix=OFPREFIX, do_save=True, hwin_x0y0=(0, 10))
def proc_data(**kwargs):
logger.info(str_kwargs(kwargs, title='Input parameters:'))
IFNAME = kwargs.get(
'ifname',
'/reg/g/psdm/detector/data_test/hdf5/amox27716-r0100-e060000-single-node.h5'
)
EVSKIP = kwargs.get('evskip', 0)
EVENTS = kwargs.get('events', 10) + EVSKIP
OFPREFIX = kwargs.get('ofprefix', './')
VERBOSE = kwargs.get('verbose', False)
proc = DLDProcessor(**kwargs)
stats = DLDStatistics(proc, **kwargs)
f = open_input_h5file(IFNAME, **kwargs)
print(' file: %s\n number of events in file %d' %
(IFNAME, f.events_in_h5file()))
t0_sec = time()
nev = 0
while f.next_event():
nev = f.event_number()
if nev < EVSKIP: continue
if nev > EVENTS: break
if do_print(nev): logger.info('Event %3d' % nev)
nhits, pktsec = f.peak_arrays()
if VERBOSE:
print_ndarr(nhits, ' number_of_hits: ')
print_ndarr(pktsec, ' peak_times_sec: ', last=4)
proc.event_proc(nev, nhits, pktsec)
stats.fill_data(nhits, pktsec)
if VERBOSE:
for i, (x, y, r,
t) in enumerate(proc.xyrt_list(nev, nhits, pktsec)):
print(' hit:%2d x:%7.3f y:%7.3f t:%10.5g r:%7.3f' %
(i, x, y, t, r))
dt = time() - t0_sec
print('%d events processing time = %.3f sec or %.6f sec/event or %.3f Hz' %
(nev, dt, dt / nev, nev / dt))
draw_plots(stats, prefix=OFPREFIX, do_save=True, hwin_x0y0=(0, 10))
def proc_data(**kwargs):
logger.info(str_kwargs(kwargs, title='Input parameters:'))
DSNAME = kwargs.get(
'dsname',
'/reg/g/psdm/detector/data2_test/xtc/data-amox27716-r0100-acqiris-e000100.xtc2'
)
DETNAME = kwargs.get('detname', 'tmo_quadanode')
EVSKIP = kwargs.get('evskip', 0)
EVENTS = kwargs.get('events', 10) + EVSKIP
VERBOSE = kwargs.get('verbose', False)
ds = DataSource(files=DSNAME)
orun = next(ds.runs())
print('\nruninfo expt: %s runnum: %d' % (orun.expt, orun.runnum))
det = orun.Detector(DETNAME)
#kwargs['detobj'] = det
kwargs['consts'] = det.calibconst
peaks = WFPeaks(**kwargs)
proc = DLDProcessor(**kwargs) #detobj=det to get cfg/calib constants
for nev, evt in enumerate(orun.events()):
if nev < EVSKIP: continue
if nev > EVENTS: break
if do_print(nev): logger.info('Event %3d' % nev)
t0_sec = time()
wts = det.raw.times(evt)
wfs = det.raw.waveforms(evt)
nhits, pkinds, pkvals, pktsec = peaks(wfs, wts) # ACCESS TO PEAK INFO
if VERBOSE:
print(" waveforms processing time = %.6f sec" % (time() - t0_sec))
print_ndarr(wfs, ' waveforms : ', last=4)
print_ndarr(wts, ' times : ', last=4)
print_ndarr(nhits, ' number_of_hits : ')
print_ndarr(pktsec, ' peak_times_sec : ', last=4)
for i, (x, y, r, t) in enumerate(proc.xyrt_list(nev, nhits, pktsec)):
print(' hit:%2d x:%7.3f y:%7.3f t:%10.5g r:%7.3f' %
(i, x, y, t, r))
def proc_data(**kwargs):
logger.info(str_kwargs(kwargs, title='Input parameters:'))
DSNAME = kwargs.get(
'dsname',
'/reg/g/psdm/detector/data2_test/xtc/data-amox27716-r0100-acqiris-e001000.xtc2'
)
DETNAME = kwargs.get('detname', 'tmo_quadanode')
EVSKIP = kwargs.get('evskip', 0)
EVENTS = kwargs.get('events', 10) + EVSKIP
EXP = kwargs.get('exp', 'amox27716')
RUN = kwargs.get('run', 85)
VERBOSE = kwargs.get('verbose', True)
PLOT = kwargs.get('plot', False)
OFPREFIX = kwargs.get('ofprefix', './')
PARAMSCFD = kwargs.get('paramsCFD')
NUMCHS = kwargs.get('numchs', 5)
NUMHITS = kwargs.get('numhits', 16)
ds = DataSource(files=DSNAME)
orun = next(ds.runs())
det = orun.Detector(DETNAME)
dldpars = {'consts': det.calibconst}
# Update calibcfg and calibtab
kwargs.update(dldpars)
tb_sec = time()
nev = 0
# Counts no. of channels with different no. peaks from CFD(raw) and CFD(win)
cn_match = 0
cn_unmatch = 0
cn_match_dld = 0
cn_unmatch_dld = 0
# Summing differences between each peak from CFD(raw) and CFD(win)
sum_err = 0
sum_pks = 0
# Summing differences between x,y,r,t (output from Roentdek)
sum_err_dld = np.zeros([EVENTS, 4], dtype=np.float64)
# Initialize PyCFD per channel
cfds = {}
for i_chan in range(NUMCHS):
CFD_params = PARAMSCFD[i_chan]
cfds[i_chan] = PyCFD(CFD_params)
# This is used for calculate Roentdek algorithm (_c is for checking
# peaks from simulated fex data).
proc = DLDProcessor(**kwargs)
proc_c = DLDProcessor(**kwargs)
# Initialize dgrampy and generate starting transitions for xtc2
out_i_chan = 4
out_xtcfname = f"data-r0085-s{str(out_i_chan).zfill(3)}-c000.xtc2"
segment_id = 4
ts = 0
data_names, datadef = dgrampy_start(EXP, RUN, out_xtcfname, segment_id, ts)
ts += 4 # Configure, BeginRun, BeginStep, Enable
for nev, evt in enumerate(orun.events()):
if nev < EVSKIP: continue
if nev >= EVENTS: break
if do_print(nev): logger.info('Event %4d' % nev)
t0_sec = time()
wts = det.raw.times(evt)
wfs = det.raw.waveforms(evt)
# Function peaks returns `pktsec` for each channel. We need to locate
# index of these peaks in wts. The indices will be used to identify
# windows of waveform wfs and startpos in wts.
NUMCHS = wfs.shape[0]
window_size = 8
nhits = np.zeros(NUMCHS, dtype=np.int64)
pktsec = np.zeros([NUMCHS, NUMHITS], dtype=wts.dtype)
nhits_fex = np.zeros(NUMCHS, dtype=np.int64)
pktsec_fex = np.zeros([NUMCHS, NUMHITS], dtype=wts.dtype)
for i_chan in range(NUMCHS):
# Find peaks using CFD
CFD_params = PARAMSCFD[i_chan]
CFD = cfds[i_chan]
pktsec_chan = CFD.CFD(wfs[i_chan, :], wts[i_chan, :])
nhits_chan = min(len(pktsec_chan), NUMHITS)
nhits[i_chan] = nhits_chan
pktsec[i_chan, :nhits_chan] = pktsec_chan[:nhits_chan]
# Calculate sample interval
sample_intervals = wts[i_chan, 1:] - wts[i_chan, :-1]
# Find peak indices
peak_ind = np.searchsorted(wts[i_chan, :], pktsec_chan)
if False:
plt.plot(wts[i_chan, :], wfs[i_chan, :], label='waveform')
# Get peak values from found indices
pktval = wfs[i_chan, peak_ind]
plt.scatter(pktsec_chan,
pktval,
marker='o',
c='r',
label=f'CFD peaks #{nhits[i_chan]}')
plt.scatter(
wts[i_chan, peak_ind],
pktval,
marker='x',
c='g',
label=f'ts from found indices #{len(wts[i_chan, peak_ind])}'
)
plt.legend()
plt.show()
# Find peak windows
pkwin_list, startpos_list, result_peaks = get_window_from_peaks(
wfs[i_chan, :],
wts[i_chan, :],
peak_ind,
window_size,
plot=PLOT,
CFD=CFD,
sample_period=CFD_params['sample_interval'],
threshold=CFD_params['threshold'])
# Save hits and peaks from fex data
nhits_chan_fex = min(len(result_peaks), NUMHITS)
nhits_fex[i_chan] = nhits_chan_fex
pktsec_fex[i_chan, :nhits_chan_fex] = result_peaks[:nhits_chan_fex]
# Write out one channel at a time (change out_i_chan to switch file)
if i_chan == out_i_chan:
dgrampy_adddata(data_names, datadef, startpos_list, pkwin_list,
ts)
ts += 1
# Calculate the differences betwen CFD(raw) and CFD(windows)
if len(result_peaks) == nhits[i_chan]:
sum_err += np.sum(
np.absolute(
np.asarray(pktsec_chan) - np.asarray(result_peaks)))
sum_pks += nhits[i_chan]
cn_match += 1
else:
cn_unmatch += 1
# end for i_chan...
# Test RoenDek algorithm
proc.event_proc(nev, nhits, pktsec)
proc_c.event_proc(nev, nhits_fex, pktsec_fex)
xyrt = proc.xyrt_list(nev, nhits, pktsec)
xyrt_c = proc_c.xyrt_list(nev, nhits_fex, pktsec_fex)
if len(xyrt) == len(xyrt_c):
for i, ((x, y, r, t), (x_c, y_c, r_c,
t_c)) in enumerate(zip(xyrt, xyrt_c)):
dx = abs(x - x_c)
dy = abs(y - y_c)
dr = abs(r - r_c)
dt = abs(t - t_c)
sum_err_dld[nev, :] = [dx, dy, dr, dt]
print(' ev:%4d hit:%2d dx:%7.3f dy:%7.3f dt:%10.5g dr:%7.3f' %
(nev, i, dx, dy, dt, dr))
cn_match_dld += 1
else:
cn_unmatch_dld += 1
if VERBOSE:
print(" ev:%4d waveforms processing time = %.6f sec" %
(nev, time() - t0_sec))
print_ndarr(wfs, ' waveforms : ', last=4)
print_ndarr(wts, ' times : ', last=4)
print_ndarr(nhits, ' number_of_hits : ')
# end for nev, evt in...
dgrampy_done(ts)
print("Summary ev:%4d processing time = %.6f sec" % (nev, time() - tb_sec))
print(
f"average err per channel: {sum_err/sum_pks} #unmatch: {cn_unmatch} ({(cn_unmatch/(5*nev))*100:.2f} %)"
)
print(
f"average err(x,y,r,t) per evt: {np.sum(sum_err_dld, axis=0)/nev} #unmatch: {cn_unmatch_dld} ({(cn_unmatch_dld/nev)*100:.2f} %)"
)
def proc_data(**kwargs):
XTCDIR = kwargs.get('xtcdir', '/cds/home/m/monarin/psana-nersc/psana2/dgrampy/amox27716')
DETNAME = kwargs.get('detname','tmo_quadanode')
EVSKIP = kwargs.get('evskip', 0)
EVENTS = kwargs.get('events', 10) + EVSKIP
EXP = kwargs.get('exp', 'amox27716')
RUN = kwargs.get('run', 85)
VERBOSE = kwargs.get('verbose', True)
PLOT = kwargs.get('plot', False)
OFPREFIX = kwargs.get('ofprefix','./')
PARAMSCFD = kwargs.get('paramsCFD')
NUMCHS = kwargs.get('numchs', 5)
NUMHITS = kwargs.get('numhits', 16)
TESTMODE = kwargs.get('testmode', 1)
ROENTDEK = kwargs.get('roentdek', 'dldproc')
MONITOR = kwargs.get('monitor', False)
DETECTORS = kwargs.get('detectors', 1)
# Open datasource
ds = DataSource(exp=EXP,
run=RUN,
dir=XTCDIR,
max_events=EVENTS,
monitor=MONITOR,
)
run = next(ds.runs())
det = run.Detector("tmo_quadanode")
# Hidden-rank problem
if det:
# Update calibration constants
dldpars = {'consts':det.calibconst}
kwargs.update(dldpars)
# Initialize PyCFD per channel
cfds = {}
for i_chan in range(NUMCHS):
cfds[i_chan] = PyCFD(PARAMSCFD[i_chan])
# Intitialize Roentdek wrapper and Xiang's hitfinder
proc = DLDProcessor(**kwargs)
HF = HitFinder(kwargs)
if PLOT: stats = DLDStatistics(proc,**kwargs)
# Collect batch rate with t0 and t1
t0 = time.monotonic()
# Find peaks for each channel and saves them in a fixed
# size array for running Roentdek algorithms
cn_no_peaks_events = 0
cn_peaks = np.zeros(EVENTS, dtype=np.int64)
dt_pks = np.zeros(EVENTS, dtype=np.float64)
dt_rtks = np.zeros(EVENTS, dtype=np.float64)
# Aux arrays for passing N-channels data to Roentdek
nhits_fex = np.zeros(NUMCHS * DETECTORS, dtype=np.int64)
pktsec_fex = np.zeros([NUMCHS * DETECTORS, NUMHITS], dtype=np.float64)
# Setup srv nodes if writing is requested
if TESTMODE == 4:
smd_batch_size = 1000
smd = ds.smalldata(filename='/cds/data/drpsrcf/users/monarin/amox27716/out/mysmallh5.h5',
batch_size=smd_batch_size,
)
for i_ev, evt in enumerate(run.events()):
if VERBOSE: print(f'EVENT: {i_ev}')
# Get list of waveforms and startpos by segment id
waveforms = det.fex.waveforms(evt, n_dets=DETECTORS)
times = det.fex.times(evt, n_dets=DETECTORS)
# Run peakfinder
# For > 1 detector, we get new detector at every NUMCHS streams.
# Note that we reuse PyCFD for all detectors (for testing).
t0_pk = time.monotonic()
if TESTMODE in (2, 3, 4):
n_result_peaks = np.zeros(NUMCHS * DETECTORS, dtype=np.int64)
for i_det in range(DETECTORS):
for i_chan in range(NUMCHS):
seg_id = int(i_det * NUMCHS + i_chan)
# TODO: Eliminate sample_interval
result_peaks = find_peaks(waveforms[seg_id],
times[seg_id],
cfds[i_chan],
)
n_result_peaks[seg_id] = len(result_peaks)
if VERBOSE: print(f' det: {i_det} chan:{i_chan} #peaks:{len(result_peaks)}')
# Save hits and peaks from fex data
nhits_chan_fex = min(len(result_peaks), NUMHITS)
nhits_fex[seg_id] = nhits_chan_fex
pktsec_fex[seg_id,:nhits_chan_fex] = result_peaks[:nhits_chan_fex]
pktsec_fex[seg_id,nhits_chan_fex:] = 0
if np.all(n_result_peaks==0): cn_no_peaks_events += 1
cn_peaks[i_ev] = np.prod(n_result_peaks + 1)
t1_pk = time.monotonic()
dt_pks[i_ev] = t1_pk - t0_pk
# Run Roentdek
t0_rtk = time.monotonic()
rt_hits = np.zeros([DETECTORS, NUMHITS, 4], dtype=np.float64)
if TESTMODE in (3, 4) and np.sum(n_result_peaks) > 0:
for i_det in range(DETECTORS):
st = int(i_det * NUMCHS)
en = st + NUMCHS
if ROENTDEK == 'dldproc':
# Roentdek wrapper
# TODO: With Mikhail - eliminate i_ev
proc.event_proc(i_ev, nhits_fex[st:en], pktsec_fex[st:en, :])
for i,(x,y,r,t) in enumerate(proc.xyrt_list(i_ev, nhits_fex[st:en], pktsec_fex[st:en,:])):
# Allow saving upto NUMHITS
if i == NUMHITS: break
rt_hits[i_det, i, :] = [x,y,r,t]
if VERBOSE: print(' hit:%2d x:%7.3f y:%7.3f t:%10.5g r:%7.3f' % (i,x,y,t,r))
else:
# Xiang's peakfinding
HF.FindHits(pktsec_fex[4,:nhits_fex[4]],
pktsec_fex[0,:nhits_fex[0]],
pktsec_fex[1,:nhits_fex[1]],
pktsec_fex[2,:nhits_fex[2]],
pktsec_fex[3,:nhits_fex[3]])
xs1,ys1,ts1 = HF.GetXYT()
if VERBOSE: print(f'xs1={xs1} ys1={ys1} ts1={ts1}')
if PLOT: stats.fill_data(nhits_fex[st:en], pktsec_fex[st:en, :])
t1_rtk = time.monotonic()
dt_rtks[i_ev] = t1_rtk - t0_rtk
# Calculate batch rate
if i_ev % 1000 == 0 and i_ev > 0:
t1 = time.monotonic()
print(f'RANK: {rank} RATE: {(1000/(t1-t0))*1e-3:.2f} kHz')
t0 = time.monotonic()
if TESTMODE == 4:
smd.event(evt, peaks=pktsec_fex, hits=rt_hits)
# end for i_ev in...
if TESTMODE == 4:
smd.done()
if PLOT:
draw_plots(stats, prefix=OFPREFIX+EXP+f'-r{str(RUN).zfill(4)}', do_save=True, hwin_x0y0=(0,10))
# Plot correlation between calc. time and #peaks and histogram of calc.time
#plt.subplot(2,2,1)
#plt.hist(dt_pks*1e3)
#plt.title('Histogram of peak finding time (ms)')
#plt.subplot(2,2,2)
#plt.scatter(cn_peaks, dt_pks*1e3)
#plt.title('CC of #peaks and peakfinding time (ms)')
#plt.subplot(2,2,3)
#plt.hist(dt_rtks*1e3)
#plt.title('Histogram of Roentdek calc. time (ms)')
#plt.subplot(2,2,4)
#plt.scatter(cn_peaks, dt_rtks*1e3)
#plt.title('CC of #np.prod(peaks+1) and Roentdek calc. time (ms)')
#plt.show()
#ind = cn_peaks == 0
#plt.subplot(2,1,1)
#plt.plot(dt_rtks[ind])
#plt.title('Roentdek Calc. Time (s.) for Zero-peak events')
#plt.subplot(2,1,2)
#plt.plot(cn_peaks[ind])
#plt.show()
if VERBOSE: print(f'EVENTS={EVENTS} NO PEAK EVENTS={cn_no_peaks_events}')