def calib(self, evt, **kwa) -> Array3d:
"""
Returns calibrated data array.
"""
logger.debug('opal_base.calib')
dtype = kwa.get('dtype', np.float32)
raw = self.raw(evt)
if raw is None:
logger.debug('det.raw.raw(evt) is None')
return None
arr = raw.astype(dtype, copy=True)
peds = self._pedestals()
if peds is None:
logger.debug('det.raw._pedestals() is None - return raw')
return arr
arr -= peds
gain = self._gain()
if gain is None:
logger.debug('det.raw._gain() is None - return raw-peds')
return arr
return divide_protected(arr, gain)
def _init_stage2(self, ndaraw):
if self.stage & 2: return
if self.counter < self.evstg1: return
t0_sec = time()
arr_av1 = divide_protected(self.arr_sum1, self.arr_sum0)
arr_av2 = divide_protected(self.arr_sum2, self.arr_sum0)
arr_rms = np.sqrt(arr_av2 - np.square(arr_av1))
rms_ave = arr_rms.mean()
gate_half = self.nsigma * rms_ave
logger.info('%s\nBegin stage 2 for %s after %d events' %
(80 * '_', self.detname, self.counter))
logger.info(' mean rms=%.3f x %.1f = intensity gate= +/- %.3f around pixel average intensity' %\
(rms_ave, self.nsigma, gate_half))
#print_ndarr(arr_av1, 'arr_av1')
self.gate_hi = np.minimum(arr_av1 + gate_half, self.arr1 * self.int_hi)
self.gate_lo = np.maximum(arr_av1 - gate_half, self.arr1 * self.int_lo)
self.gate_hi = np.array(self.gate_hi, dtype=ndaraw.dtype)
self.gate_lo = np.array(self.gate_lo, dtype=ndaraw.dtype)
self.arr_sum0 = np.zeros(self.shape, dtype=np.int64)
self.arr_sum1 = np.zeros(self.shape, dtype=np.double)
self.arr_sum2 = np.zeros(self.shape, dtype=np.double)
self.stage = 2
logger.info('Stage 2 initialization for %s consumes dt=%7.3f sec' %
(self.detname, time() - t0_sec))
def test_image(args):
import psana.detector.UtilsEpix10ka as ue
from psana.detector.UtilsGraphics import gr, fleximage, flexhist, fleximagespec
dograph = args.dograph.lower()
flimg, flspe, flims = None, None, None
ds, run, det = ds_run_det(args)
peds = det.raw._pedestals(
) if args.grindex is None else det.raw._pedestals()[args.grindex, :]
is_epix10ka = 'epix' in det.raw._uniqueid
dcfg = ue.config_object_epix10ka(det) if is_epix10ka else None
break_event_loop = False
nframes = 0
sum_arr, sum_sta = None, None
med_vs_evt = np.zeros(args.events - args.evskip + 10, dtype=np.float64)
nrec_med = 0
for stepnum, step in enumerate(run.steps()):
print('%s\nStep %1d' % (50 * '_', stepnum))
if args.stepsel is not None and stepnum != args.stepsel:
print(' skip - step selected in option -M is %1d' %
(args.stepsel))
continue
print('%s\n begin event loop' % (50 * '_'))
for evnum, evt in enumerate(step.events()):
if evnum < args.evskip:
print('Step %1d Event %04d - skip first %04d events' % (stepnum, evnum, args.evskip),\
end=('\r' if evnum<args.evskip-1 else '\n'))
continue
if evnum > args.events:
print('break by number of events limit %d set in option -N' %
args.events)
break_event_loop = True
break
if evnum > 2 and evnum % args.evjump != 0: continue
print('%s\nStep %1d Event %04d' % (50 * '_', stepnum, evnum))
if dcfg is not None:
s = ' gain mode fractions for: FH FM FL'\
' AHL-H AML-M AHL-L AML-L\n%s' % (29*' ')
#ue.info_pixel_gain_mode_for_fractions(dcfg, data=det.raw.raw(evt), msg=s))
gmfracs = ue.pixel_gain_mode_fractions(dcfg,
data=det.raw.raw(evt))
print(ue.info_pixel_gain_mode_for_fractions(gmfracs, msg=s))
gmind = ue.gain_mode_index_from_fractions(gmfracs)
gmname = ue.gain_mode_name_for_index(gmind).upper()
print(' == major gain mode %d : %s' % (gmind, gmname))
#print(' == gain mode: %s' % ue.find_gain_mode(dcfg, data=None).upper())
if peds is None: peds = det.raw._pedestals()[gmind, :]
#user_mask = np.ones_like(det.raw.raw(evt), dtype=DTYPE_MASK) #np.uint8
#user_mask[0,100:150,200:250] = 0
user_mask = None
arr = None
if args.show == 'raw-peds-med':
arr = (det.raw.raw(evt) & 0x3fff) - peds
med = np.median(arr)
print('XXX from raw-peds subtract its median = %.3f' % med)
arr -= med
if arr is None:
arr = det.raw.calib(evt, cmpars=(7,7,100,10),\
mbits=0o7, mask=user_mask, edge_rows=10, edge_cols=10, center_rows=5, center_cols=5)\
if args.show == 'calibcm' else\
det.raw.calib(evt, cmpars=(8,7,10,10))\
if args.show == 'calibcm8' else\
det.raw.calib(evt) if args.show == 'calib' else\
peds if args.show == 'peds' else\
det.raw._gain_range_index(evt) if args.show == 'grind' else\
(det.raw.raw(evt) & 0x3fff) - peds if args.show == 'raw-peds' else\
(det.raw.raw(evt) & 0x3fff) if args.show == 'rawbm' else\
np.ones_like(det.raw.raw(evt)) if args.show == 'ones' else\
(det.raw.raw(evt) & args.bitmask)
#if args.show == 'calibcm': arr += 1 # to see panel edges
logger.info(info_ndarr(arr, 'arr '))
if arr is None: continue
med = np.median(arr)
med_vs_evt[nrec_med] = med
nrec_med += 1
if args.cumulat:
if (med > args.thrmin) and (med < args.thrmax):
nframes += 1
cond = arr > args.thrpix
if nframes != 1:
_ = np.add(sum_arr[cond], arr[cond], out=sum_arr[cond])
sum_sta[cond] += 1
else:
sum_arr = np.array(arr, dtype=np.float64)
sum_sta = np.zeros_like(arr, dtype=np.uint64)
#if nframes > 1: arr = sum_arr/float(nframes)
if nframes > 1: arr = divide_protected(sum_arr, sum_sta)
print('Step %1d event:%04d nframes:%04d arr median:%.3f' %
(stepnum, evnum, nframes, med))
else:
continue
t0_sec = time()
img = det.raw.image(evt,
nda=arr,
pix_scale_size_um=args.pscsize,
mapmode=args.mapmode)
print('image composition time = %.6f sec ' % (time() - t0_sec))
logger.info(info_ndarr(img, 'img '))
logger.info(info_ndarr(arr, 'arr '))
if img is None: continue
title = '%s %s run:%s ev:%d' % (args.detname, args.expname,
args.runs, evnum)
if 'i' in dograph:
if flimg is None:
flimg = fleximage(img, arr=arr, fraclo=0.05, frachi=0.95)
flimg.move(10, 20)
else:
flimg.update(img, arr=arr)
flimg.fig.canvas.set_window_title(title)
flimg.axtitle(title)
if 'h' in dograph:
if flspe is None:
flspe = flexhist(arr,
bins=50,
color='green',
fraclo=0.001,
frachi=0.999)
flspe.move(800, 20)
else:
flspe.update(arr,
bins=50,
color='green',
fraclo=0.001,
frachi=0.999)
flspe.fig.canvas.set_window_title(title)
flspe.axtitle(title)
if 'c' in dograph:
if flims is None:
flims = fleximagespec(img, arr=arr, bins=100, color='lightgreen',\
amin=args.gramin, amax=args.gramax,\
nneg=args.grnneg, npos=args.grnpos,\
fraclo=args.grfrlo, frachi=args.grfrhi,\
)
flims.move(10, 20)
else:
#print(info_ndarr(arr, 'YYY before update arr: ', last=5))
flims.update(img, arr=arr)
flims.axtitle(title)
gr.show(mode=1)
if break_event_loop: break
med_vs_evt = med_vs_evt[:nrec_med]
med = np.median(med_vs_evt)
q05 = np.quantile(med_vs_evt, 0.05, interpolation='linear')
q95 = np.quantile(med_vs_evt, 0.95, interpolation='linear')
print(info_ndarr(med_vs_evt, 'per event median ', last=nrec_med - 1))
print(' median over %d event-records: %.3f' % (nrec_med, med))
print(' quantile(med_vs_evt, 0.05): %.3f' % q05)
print(' quantile(med_vs_evt, 0.95): %.3f' % q95)
if args.dograph:
print(
'\n !!! TO EXIT - close graphical window(s) - click on [x] in the window corner'
)
gr.show()
if args.ofname is not None:
if 'i' in dograph:
gr.save_fig(flimg.fig, fname=args.ofname + '-img', verb=True)
if 'h' in dograph:
gr.save_fig(flspe.fig, fname=args.ofname + '-spe', verb=True)
if 'c' in dograph:
gr.save_fig(flims.fig,
fname=args.ofname + '-imgspe',
verb=True)
print(50 * '-')
def summary(self, evt):
logger.info('%s\nRaw data for %s found/selected in %d events' %
(80 * '_', self.detname, self.counter))
if self.counter:
logger.info(', begin data summary stage')
else:
logger.info(', STOP processing, there are no arrays to save...')
return
t0_sec = time()
# make shorter references
det = self.det
ofname = self.ofname
plotim = self.plotim
savebw = self.savebw
int_hi = self.int_hi
int_lo = self.int_lo
fraclm = self.fraclm
counter = self.counter
arr_av1 = divide_protected(self.arr_sum1, self.arr_sum0)
arr_av2 = divide_protected(self.arr_sum2, self.arr_sum0)
frac_int_lo = np.array(self.sta_int_lo / counter, dtype=np.float32)
frac_int_hi = np.array(self.sta_int_hi / counter, dtype=np.float32)
arr_rms = np.sqrt(arr_av2 - np.square(arr_av1))
rms_min, rms_max = evaluate_limits(arr_rms,
self.rmsnlo,
self.rmsnhi,
self.rms_lo,
self.rms_hi,
cmt='RMS')
ave_min, ave_max = evaluate_limits(arr_av1,
self.intnlo,
self.intnhi,
self.int_lo,
self.int_hi,
cmt='AVE')
arr_sta_rms_hi = np.select((arr_rms > rms_max, ), (self.arr1, ), 0)
arr_sta_rms_lo = np.select((arr_rms < rms_min, ), (self.arr1, ), 0)
arr_sta_int_hi = np.select((frac_int_hi > fraclm, ), (self.arr1, ), 0)
arr_sta_int_lo = np.select((frac_int_lo > fraclm, ), (self.arr1, ), 0)
arr_sta_ave_hi = np.select((arr_av1 > ave_max, ), (self.arr1, ), 0)
arr_sta_ave_lo = np.select((arr_av1 < ave_min, ), (self.arr1, ), 0)
logger.info(' Bad pixel status:')
logger.info(' status 1: %8d pixel rms > %.3f' %
(arr_sta_rms_hi.sum(), rms_max))
logger.info(' status 8: %8d pixel rms < %.3f' %
(arr_sta_rms_lo.sum(), rms_min))
logger.info(
' status 2: %8d pixel intensity > %g in more than %g fraction of events'
% (arr_sta_int_hi.sum(), int_hi, fraclm))
logger.info(
' status 4: %8d pixel intensity < %g in more than %g fraction of events'
% (arr_sta_int_lo.sum(), int_lo, fraclm))
logger.info(' status 16: %8d pixel average > %g' %
(arr_sta_ave_hi.sum(), ave_max))
logger.info(' status 32: %8d pixel average < %g' %
(arr_sta_ave_lo.sum(), ave_min))
#0/1/2/4/8/16/32 for good/hot-rms/saturated/cold/cold-rms/average above limit/average below limit,
arr_sta = np.zeros(self.shape, dtype=np.int64)
arr_sta += arr_sta_rms_hi # hot rms
arr_sta += arr_sta_rms_lo * 8 # cold rms
arr_sta += arr_sta_int_hi * 2 # satturated
arr_sta += arr_sta_int_lo * 4 # cold
arr_sta += arr_sta_ave_hi * 16 # too large average
arr_sta += arr_sta_ave_lo * 32 # too small average
arr_msk = np.select((arr_sta > 0, ), (self.arr0, ), 1)
cmod = None # self._common_mode_pars(arr_av1, arr_rms, arr_msk)
if plotim & 1: plot_det_image(det, evt, arr_av1, tit='average')
if plotim & 2: plot_det_image(det, evt, arr_rms, tit='RMS')
if plotim & 4: plot_det_image(det, evt, arr_sta, tit='status')
if plotim & 8: plot_det_image(det, evt, arr_msk, tit='mask')
if plotim & 16: plot_det_image(det, evt, self.arr_max, tit='maximum')
if plotim & 32: plot_det_image(det, evt, self.arr_min, tit='minimum')
if plotim & 64:
plot_det_image(det,
evt,
self.sta_int_lo,
tit='statistics below threshold')
if plotim & 128:
plot_det_image(det,
evt,
self.sta_int_hi,
tit='statistics above threshold')
cmts = ['DATASET %s' % self.detname, 'STATISTICS %d' % counter]
# Save n-d array in text file %
template = fname_template(self.orun, self.detname, ofname, counter,
self.ev1_sec, self.ev1_nsec)
addmetad = True
verbos = self.verbos
if savebw & 1:
save_txt(template % 'ave', arr_av1, cmts + ['ARR_TYPE average'],
'%8.2f', verbos, addmetad)
if savebw & 2:
save_txt(template % 'rms', arr_rms, cmts + ['ARR_TYPE RMS'],
'%8.2f', verbos, addmetad)
if savebw & 4:
save_txt(template % 'sta', arr_sta, cmts + ['ARR_TYPE status'],
'%d', verbos, addmetad)
if savebw & 8:
save_txt(template % 'msk', arr_msk, cmts + ['ARR_TYPE mask'],
'%1d', verbos, addmetad)
if savebw & 16:
save_txt(template % 'max', self.arr_max, cmts + ['ARR_TYPE max'],
'%d', verbos, addmetad)
if savebw & 32:
save_txt(template % 'min', self.arr_min, cmts + ['ARR_TYPE min'],
'%d', verbos, addmetad)
if savebw & 64 and cmod is not None:
np.savetxt(template % 'cmo',
cmod,
fmt='%d',
delimiter=' ',
newline=' ')
save_txt(template % 'cmm', cmod, cmts + ['ARR_TYPE common_mode'],
'%d', verbos, False)
args = vars(self.args)
version = args.get('version', str_tstamp('V%Y%m%dT%H%M%S', t0_sec))
orun = self.orun
loglev = args.get('loglev', 'INFO')
# args.expnam # args.runnum, orun.timestamp
kwa = {
'detname': self.detname,
'experiment': orun.expt,
'runnum': orun.runnum,
'timestamp': str_tstamp(time_sec=self.ev1_sec),
'version': version,
'comment': str(cmts),
'loglev': loglev,
}
logger.debug('kwa:', kwa)
logger.debug('args:', self.args)
if self.addcdb:
from psana.pscalib.calib.MDBCommands import add_constants
if savebw & 1: add_constants(template % 'ave', 'pedestals', **kwa)
if savebw & 2: add_constants(template % 'rms', 'pixel_rms', **kwa)
if savebw & 4:
add_constants(template % 'sta', 'pixel_status', **kwa)
if savebw & 8: add_constants(template % 'msk', 'pixel_mask', **kwa)
if savebw & 16: add_constants(template % 'max', 'pixel_max', **kwa)
if savebw & 32: add_constants(template % 'min', 'pixel_min', **kwa)
logger.info('Data summary for %s is completed, dt=%7.3f sec' %
(self.detname, time() - t0_sec))
def calib_epix10ka_any(det_raw, evt, cmpars=None, **kwa): #cmpars=(7,2,100)):
"""
Returns calibrated epix10ka data
- gets constants
- gets raw data
- evaluates (code - pedestal - offset)
- applys common mode correction if turned on
- apply gain factor
Parameters
- det_raw (psana.Detector.raw) - Detector.raw object
- evt (psana.Event) - Event object
- cmpars (tuple) - common mode parameters
= None - use pars from calib directory
= cmpars=(<alg>, <mode>, <maxcorr>)
alg is not used
mode =0-correction is not applied, =1-in rows, =2-in cols-WORKS THE BEST
i.e: cmpars=(7,0,100) or (7,2,100)
- **kwa - used here and passed to det_raw.mask_comb
- nda_raw - substitute for det_raw.raw(evt)
- mbits - parameter of the det_raw.mask_comb(...)
- mask - user defined mask passed as optional parameter
"""
logger.debug('In calib_epix10ka_any')
t0_sec_tot = time()
nda_raw = kwa.get('nda_raw', None)
raw = det_raw.raw(
evt
) if nda_raw is None else nda_raw # shape:(352, 384) or suppose to be later (<nsegs>, 352, 384) dtype:uint16
if raw is None: return None
_cmpars = det_raw._common_mode() if cmpars is None else cmpars
gain = det_raw._gain() # - 4d gains (7, <nsegs>, 352, 384)
peds = det_raw._pedestals() # - 4d pedestals
if gain is None: return None # gain = np.ones_like(peds) # - 4d gains
if peds is None: return None # peds = np.zeros_like(peds) # - 4d gains
if store.gfac is None:
# do ONCE this initialization
logger.debug(info_ndarr(raw, '\n raw ')\
+info_ndarr(gain, '\n gain')\
+info_ndarr(peds, '\n peds'))
store.gfac = divide_protected(np.ones_like(gain), gain)
store.arr1 = np.ones_like(raw, dtype=np.int8)
logger.debug(info_ndarr(store.gfac, '\n gfac '))
# 'FH','FM','FL','AHL-H','AML-M','AHL-L','AML-L'
#store.gf4 = np.ones_like(raw, dtype=np.int32) * 0.25 # 0.3333 # M - perefierial
#store.gf6 = np.ones_like(raw, dtype=np.int32) * 1 # L - center
gfac = store.gfac
if store.dcfg is None: store.dcfg = config_object_epix10ka_raw(det_raw)
gmaps = gain_maps_epix10ka_any(store.dcfg, raw)
if gmaps is None: return None
gr0, gr1, gr2, gr3, gr4, gr5, gr6 = gmaps
factor = np.select(gmaps,\
(gfac[0,:], gfac[1,:], gfac[2,:], gfac[3,:],\
gfac[4,:], gfac[5,:], gfac[6,:]), default=1) # 2msec
pedest = np.select(gmaps,\
(peds[0,:], peds[1,:], peds[2,:], peds[3,:],\
peds[4,:], peds[5,:], peds[6,:]), default=0)
store.counter += 1
if not store.counter % 100:
logger.debug(info_gain_mode_arrays(gmaps))
logger.debug(info_pixel_gain_mode_statistics(gmaps))
logger.debug('TOTAL consumed time (sec) = %.6f' % (time() - t0_sec_tot))
logger.debug(info_ndarr(factor, 'calib_epix10ka factor'))
logger.debug(info_ndarr(pedest, 'calib_epix10ka pedest'))
arrf = np.array(raw & M14, dtype=np.float32) - pedest
logger.debug('common-mode correction pars cmp: %s' % str(_cmpars))
if store.mask is None:
mbits = kwa.pop('mbits', 1) # 1-mask from status, etc.
mask = det_raw._mask_comb(mbits=mbits, **kwa) if mbits > 0 else None
mask_opt = kwa.get(
'mask',
None) # mask optional parameter in det_raw.calib(...,mask=...)
store.mask = mask if mask_opt is None else mask_opt if mask is None else merge_masks(
mask, mask_opt)
mask = store.mask if store.mask is not None else np.ones_like(
raw, dtype=DTYPE_MASK)
if _cmpars is not None:
alg, mode, cormax = int(_cmpars[0]), int(_cmpars[1]), _cmpars[2]
npixmin = _cmpars[3] if len(_cmpars) > 3 else 10
if mode > 0:
t0_sec_cm = time()
#t2_sec_cm = time()
arr1 = store.arr1 # np.ones_like(mask, dtype=np.uint8)
grhm = np.select(
(gr0, gr1, gr3, gr4),
(arr1, arr1, arr1, arr1), default=0) if alg == 7 else arr1
gmask = np.bitwise_and(grhm, mask) if mask is not None else grhm
#logger.debug(info_ndarr(arr1, '\n arr1'))
#logger.debug(info_ndarr(grhm, 'XXXX grhm'))
#logger.debug(info_ndarr(gmask, 'XXXX gmask'))
#logger.debug('common-mode mask massaging (sec) = %.6f' % (time()-t2_sec_cm)) # 5msec
logger.debug(info_ndarr(gmask, 'gmask')\
+ '\n per panel statistics of cm-corrected pixels: %s' % str(np.sum(gmask, axis=(1,2), dtype=np.uint32)))
#sh = (nsegs, 352, 384)
hrows = 176 # int(352/2)
for s in range(arrf.shape[0]):
if mode & 4: # in banks: (352/2,384/8)=(176,48) pixels
common_mode_2d_hsplit_nbanks(arrf[s, :hrows, :],
mask=gmask[s, :hrows, :],
nbanks=8,
cormax=cormax,
npix_min=npixmin)
common_mode_2d_hsplit_nbanks(arrf[s, hrows:, :],
mask=gmask[s, hrows:, :],
nbanks=8,
cormax=cormax,
npix_min=npixmin)
if mode & 1: # in rows per bank: 384/8 = 48 pixels # 190ms
common_mode_rows_hsplit_nbanks(arrf[s, ],
mask=gmask[s, ],
nbanks=8,
cormax=cormax,
npix_min=npixmin)
if mode & 2: # in cols per bank: 352/2 = 176 pixels # 150ms
common_mode_cols(arrf[s, :hrows, :],
mask=gmask[s, :hrows, :],
cormax=cormax,
npix_min=npixmin)
common_mode_cols(arrf[s, hrows:, :],
mask=gmask[s, hrows:, :],
cormax=cormax,
npix_min=npixmin)
logger.debug('TIME common-mode correction = %.6f sec for cmp=%s' %
(time() - t0_sec_cm, str(_cmpars)))
return arrf * factor if mask is None else arrf * factor * mask # gain correction
def summary(self, evt):
logger.info('%s\nRaw data for %s found/selected in %d events' %
(80 * '_', self.detname, self.counter))
if self.counter:
logger.info(', begin data summary stage')
else:
logger.info(', STOP processing, there are no arrays to save...')
return
t0_sec = time()
# make shorter references
det = self.det
ofname = self.ofname
plotim = self.plotim
savebw = self.savebw
verbos = self.verbos
int_hi = self.int_hi
int_lo = self.int_lo
fraclm = self.fraclm
counter = self.counter
arr_av1 = divide_protected(self.arr_sum1, self.arr_sum0)
arr_av2 = divide_protected(self.arr_sum2, self.arr_sum0)
frac_int_lo = np.array(self.sta_int_lo / counter, dtype=np.float32)
frac_int_hi = np.array(self.sta_int_hi / counter, dtype=np.float32)
arr_rms = np.sqrt(arr_av2 - np.square(arr_av1))
rms_min, rms_max = evaluate_limits(arr_rms,
self.rmsnlo,
self.rmsnhi,
self.rms_lo,
self.rms_hi,
self.verbos,
cmt='RMS')
#if self.rms_lo is None or self.rms_hi is None or self.rms_hi == 0.\
#else (self.rms_lo, self.rms_hi)
#if rms_min<0 : rms_min=0
ave_min, ave_max = evaluate_limits(arr_av1, self.intnlo, self.intnhi, self.int_lo, self.int_hi, self.verbos, cmt='AVE')\
arr_sta_rms_hi = np.select((arr_rms > rms_max, ), (self.arr1, ), 0)
arr_sta_rms_lo = np.select((arr_rms < rms_min, ), (self.arr1, ), 0)
arr_sta_int_hi = np.select((frac_int_hi > fraclm, ), (self.arr1, ), 0)
arr_sta_int_lo = np.select((frac_int_lo > fraclm, ), (self.arr1, ), 0)
arr_sta_ave_hi = np.select((arr_av1 > ave_max, ), (self.arr1, ), 0)
arr_sta_ave_lo = np.select((arr_av1 < ave_min, ), (self.arr1, ), 0)
logger.info(' Bad pixel status:')
logger.info(' status 1: %8d pixel rms > %.3f' %
(arr_sta_rms_hi.sum(), rms_max))
logger.info(' status 8: %8d pixel rms < %.3f' %
(arr_sta_rms_lo.sum(), rms_min))
logger.info(
' status 2: %8d pixel intensity > %g in more than %g fraction of events'
% (arr_sta_int_hi.sum(), int_hi, fraclm))
logger.info(
' status 4: %8d pixel intensity < %g in more than %g fraction of events'
% (arr_sta_int_lo.sum(), int_lo, fraclm))
logger.info(' status 16: %8d pixel average > %g' %
(arr_sta_ave_hi.sum(), ave_max))
logger.info(' status 32: %8d pixel average < %g' %
(arr_sta_ave_lo.sum(), ave_min))
#0/1/2/4/8/16/32 for good/hot-rms/saturated/cold/cold-rms/average above limit/average below limit,
arr_sta = np.zeros(self.shape, dtype=np.int64)
arr_sta += arr_sta_rms_hi # hot rms
arr_sta += arr_sta_rms_lo * 8 # cold rms
arr_sta += arr_sta_int_hi * 2 # satturated
arr_sta += arr_sta_int_lo * 4 # cold
arr_sta += arr_sta_ave_hi * 16 # too large average
arr_sta += arr_sta_ave_lo * 32 # too small average
arr_msk = np.select((arr_sta > 0, ), (self.arr0, ), 1)
cmod = None # self._common_mode_pars(arr_av1, arr_rms, arr_msk)
if plotim & 1: plot_det_image(det, evt, arr_av1, tit='average')
if plotim & 2: plot_det_image(det, evt, arr_rms, tit='RMS')
if plotim & 4: plot_det_image(det, evt, arr_sta, tit='status')
if plotim & 8: plot_det_image(det, evt, arr_msk, tit='mask')
if plotim & 16: plot_det_image(det, evt, self.arr_max, tit='maximum')
if plotim & 32: plot_det_image(det, evt, self.arr_min, tit='minimum')
if plotim & 64:
plot_det_image(det,
evt,
self.sta_int_lo,
tit='statistics below threshold')
if plotim & 128:
plot_det_image(det,
evt,
self.sta_int_hi,
tit='statistics above threshold')
cmts = ['DATASET %s' % self.detname, 'STATISTICS %d' % counter]
# Save n-d array in text file %
template = fname_template(self.orun, self.detname, ofname, counter,
self.ev1_sec, self.ev1_nsec)
addmetad = True
if savebw & 1:
save_txt(template % 'ave', arr_av1, cmts + ['ARR_TYPE average'],
'%8.2f', verbos, addmetad)
if savebw & 2:
save_txt(template % 'rms', arr_rms, cmts + ['ARR_TYPE RMS'],
'%8.2f', verbos, addmetad)
if savebw & 4:
save_txt(template % 'sta', arr_sta, cmts + ['ARR_TYPE status'],
'%d', verbos, addmetad)
if savebw & 8:
save_txt(template % 'msk', arr_msk, cmts + ['ARR_TYPE mask'],
'%1d', verbos, addmetad)
if savebw & 16:
save_txt(template % 'max', self.arr_max, cmts + ['ARR_TYPE max'],
'%d', verbos, addmetad)
if savebw & 32:
save_txt(template % 'min', self.arr_min, cmts + ['ARR_TYPE min'],
'%d', verbos, addmetad)
if savebw & 64 and cmod is not None:
np.savetxt(template % 'cmo',
cmod,
fmt='%d',
delimiter=' ',
newline=' ')
save_txt(template % 'cmm', cmod, cmts + ['ARR_TYPE common_mode'],
'%d', verbos, False)
if self.verbos & 1:
logger.info('Data summary for %s is completed, dt=%7.3f sec' %
(self.detname, time() - t0_sec))
def pedestals_calibration(*args, **opts):
"""NEWS significant ACCELERATION is acheived:
- accumulate data for entire epix10kam_2m/quad array
- use MPI
all-panel or selected-panel one-step (gain range) or all steps calibration of pedestals
"""
fname = opts.get('fname', None)
detname = opts.get('det', None)
exp = opts.get('exp', None)
runs = opts.get('runs', None)
nrecs = opts.get('nrecs', 1000)
stepnum = opts.get('stepnum', None)
stepmax = opts.get('stepmax', 5)
evskip = opts.get('evskip', 0)
events = opts.get('events', 1000)
dirxtc = opts.get('dirxtc', None)
dirrepo = opts.get('dirrepo', CALIB_REPO_EPIX10KA)
fmt_peds = opts.get('fmt_peds', '%.3f')
fmt_rms = opts.get('fmt_rms', '%.3f')
fmt_status = opts.get('fmt_status', '%4i')
idx_sel = opts.get('idx', None)
dirmode = opts.get('dirmode', 0o777)
filemode = opts.get('filemode', 0o666)
usesmd = opts.get('usesmd', False)
logmode = opts.get('logmode', 'DEBUG')
errskip = opts.get('errskip', False)
logger.setLevel(DICT_NAME_TO_LEVEL[logmode])
#irun = runs[0] if isinstance(runs, list) else\
# int(runs.split(',',1)[0].split('-',1)[0]) # int first run number from str of run(s)
irun = irun_first(runs)
#dsname = 'exp=%s:run=%s'%(exp,runs) if dirxtc is None else 'exp=%s:run=%s:dir=%s'%(exp, runs, dirxtc)
#if usesmd: dsname += ':smd'
_name = sys._getframe().f_code.co_name
logger.info('In %s\n exp: %s\n runs: %s\n detector: %s' % (_name, exp, str(runs), detname))
save_log_record_on_start(dirrepo, _name, dirmode)
#cpdic = get_config_info_for_dataset_detname(dsname, detname)
#tstamp = cpdic.get('tstamp', None)
#panel_ids = cpdic.get('panel_ids', None)
#expnum = cpdic.get('expnum', None)
#dettype = cpdic.get('dettype', None)
#shape = cpdic.get('shape', None)
#ny,nx = shape
#panel_id = get_panel_id(panel_ids, idx)
#logger.debug('Found panel ids:\n%s' % ('\n'.join(panel_ids)))
#read input xtc file and accumulate block of data
#================= MPI
#from mpi4py import MPI
#comm = MPI.COMM_WORLD
#rank = comm.Get_rank()
#size = comm.Get_size() # number of MPI nodes; 1 for regular python command
#=================
kwa = data_source_kwargs(**opts)
ds = DataSource(**kwa)
logger.debug('ds.runnum_list = %s' % str(ds.runnum_list))
logger.debug('ds.detectors = %s' % str(ds.detectors))
mode = None # gain_mode
nstep_tot = -1
#orun = next(ds.runs())
for orun in ds.runs():
logger.debug('==run.runnum : %d' % orun.runnum) # 27
logger.debug(' run.detnames : %s' % str(orun.detnames)) # {'epixquad'}
logger.debug(' run.expt : %s', orun.expt) # ueddaq02
runtstamp = orun.timestamp # 4193682596073796843 relative to 1990-01-01
trun_sec = seconds(runtstamp) # 1607569818.532117 sec
#tstamp = str_tstamp(time_sec=int(trun_sec)) #fmt='%Y-%m-%dT%H:%M:%S%z'
tstamp_run, tstamp_now = tstamps_run_and_now(int(trun_sec))
tstamp = tstamp_run
logger.debug(' run.timestamp: %d' % orun.timestamp)
logger.debug(' run unix epoch time %06f sec' % trun_sec)
logger.debug(' run tstamp: %s' % tstamp_run)
logger.debug(' now tstamp: %s' % tstamp_now)
det = orun.Detector(detname)
step_value = orun.Detector('step_value')
step_docstring = orun.Detector('step_docstring')
#cd = orun.Detector('ControlData') #LCLS1
logger.debug('--- det.raw._det_name: %s' % det.raw._det_name) # epixquad
logger.debug(' det.raw._dettype : %s' % det.raw._dettype) # epix
logger.debug(' det.raw._calibconst.keys(): %s' % str(det.raw._calibconst.keys())) # dict_keys(['geometry'])
#logger.debug(' det.raw._uniqueid: %s' % det.raw._uniqueid)
#logger.debug(' det.raw._sorted_segment_ids: %s' % str(det.raw._sorted_segment_ids))
#logger.debug(' ue.fullname_epix10ka_detector: %s' % ue.fullname_epix10ka_detector(det))
segment_ids = ue.segment_ids_epix10ka_detector(det)
segment_inds = ue.segment_indices_epix10ka_detector(det)
s = 'segment inds and ids in the detector'
for i,id in zip(segment_inds,segment_ids):
s += '\n seg:%02d id:%s' % (i,id)
logger.info(s)
#logger.debug(' ue.segment_ids_epix10ka_detector: %s' % str(ue.segment_ids_epix10ka_detector(det)))
#logger.debug(' ue.segment_indices_epix10ka_detector: %s' % str(ue.segment_indices_epix10ka_detector(det)))
dcfg = ue.config_object_epix10ka(det)
for nstep_run, step in enumerate(orun.steps()): #(loop through calyb cycles, using only the first):
nstep_tot += 1
logger.info('\n=============== step %2d ===============' % nstep_tot)
logger.debug(' step.evt._seconds: %d' % step.evt._seconds)
metadic = json.loads(step_docstring(step))
nstep = step_counter(metadic, nstep_tot, nstep_run, stype='pedestal')
if nstep is None: continue
#if size > 1:
# # if MPI is on process all steps, step per rank
# if nstep < rank: continue
# if nstep > rank: break
if nstep_tot>=stepmax:
logger.info('==== Step:%02d loop is terminated, --stepmax=%d' % (nstep_tot, stepmax))
break
elif stepnum is not None:
if nstep < stepnum:
logger.info('==== Step:%02d is skipped, --stepnum=%d' % (nstep, stepnum))
continue
elif nstep > stepnum:
logger.info('==== Step:%02d loop is terminated, --stepnum=%d' % (nstep, stepnum))
break
#for k,v in det.raw._seg_configs().items(): # cpo's pattern DOES NOT WORK
for k,v in dcfg.items():
scob = v.config
logger.info(info_ndarr(scob.asicPixelConfig[:,:-2,:], 'seg:%02d trbits: %s asicPixelConfig:'%(k, str(scob.trbit))))
gmaps = ue.gain_maps_epix10ka_any(dcfg, data=None)
logger.debug('gain mode statistics:' + ue.info_pixel_gain_mode_statistics(gmaps))
logger.debug(ue.info_pixel_gain_mode_fractions(dcfg, data=None, msg='gain mode fractions :'))
logger.debug('gain maps'\
+ info_ndarr(gmaps[0],'\n FH ')\
+ info_ndarr(gmaps[1],'\n FM ')\
+ info_ndarr(gmaps[2],'\n FL ')\
+ info_ndarr(gmaps[3],'\n AHL ')\
+ info_ndarr(gmaps[4],'\n AML ')\
)
mode = ue.find_gain_mode(dcfg, data=None).upper()
if mode in ue.GAIN_MODES_IN:
mode_in_step = ue.GAIN_MODES_IN[nstep]
logger.info('== step %d: dark run processing for gain mode in configuration %s and step number %s'\
%(nstep, mode, mode_in_step))
if mode != mode_in_step:
logger.warning('INCONSISTENT GAIN MODES IN CONFIGURATION AND STEP NUMBER/METADATA')
if not errskip: sys.exit()
logger.warning('FLAG ERRSKIP IS %s - keep processing assuming gain mode %s' % (errskip,mode))
#continue
else:
logger.warning('UNRECOGNIZED GAIN MODE: %s, DARKS NOT UPDATED...'%mode)
sys.exit()
#return
sh = gmaps[0].shape
shape_block = [nrecs,] + list(sh) # [nrecs, <number-of-segments>, 352, 384]
logger.info('Accumulate raw frames in block shape = %s' % str(shape_block))
block=np.zeros(shape_block,dtype=np.uint16)
nrec,nevt = -1,0
ss = None
for nevt,evt in enumerate(step.events()):
raw = det.raw.raw(evt)
do_print = selected_record(nevt)
if raw is None:
logger.info('==== Ev:%04d rec:%04d raw is None' % (nevt,nrec))
continue
if nevt < evskip:
logger.debug('==== Ev:%04d is skipped, --evskip=%d' % (nevt,evskip))
continue
elif evskip>0 and (nevt == evskip):
s = 'Events < --evskip=%d are skipped' % evskip
#print(s)
logger.info(s)
if nevt > events-1:
logger.info(ss)
logger.info('==== Ev:%04d event loop is terminated, --events=%d' % (nevt,events))
print()
break
if nrec > nrecs-2:
logger.info(ss)
logger.info('==== Ev:%04d event loop is terminated - collected sufficient number of frames, --nrecs=%d' % (nevt,nrecs))
break
else:
nrec += 1
ss = info_ndarr(raw & ue.M14, 'Ev:%04d rec:%04d raw & M14 ' % (nevt,nrec))
if do_print: logger.info(ss)
block[nrec]=(raw & ue.M14)
if nevt < events: logger.info('==== Ev:%04d end of events in run step %d' % (nevt,nstep_run))
print_statistics(nevt, nrec)
#---- process statistics in block-array for panels
for idx, panel_id in zip(segment_inds,segment_ids):
if idx_sel is not None and idx_sel != idx: continue # skip panels with inices other than idx_sel if specified
logger.info('\n%s\nprocess panel:%02d id:%s' % (96*'=', idx, panel_id))
#if mode is None:
# msg = 'Gain mode for dark processing is not defined "%s" try to set option -m <gain-mode>' % mode
# logger.warning(msg)
# sys.exit(msg)
dir_panel, dir_offset, dir_peds, dir_plots, dir_work, dir_gain, dir_rms, dir_status = dir_names(dirrepo, panel_id)
#print('XXXX panel_id, tstamp, exp, irun', panel_id, tstamp, exp, irun)
fname_prefix, panel_alias = file_name_prefix(dirrepo, panel_id, tstamp, exp, irun)
logger.debug('\n fname_prefix:%s\n panel_alias :%s' % (fname_prefix, panel_alias))
prefix_offset, prefix_peds, prefix_plots, prefix_gain, prefix_rms, prefix_status =\
path_prefixes(fname_prefix, dir_offset, dir_peds, dir_plots, dir_gain, dir_rms, dir_status)
#logger.debug('Directories under %s\n SHOULD ALREADY EXIST after charge-injection offset_calibration' % dir_panel)
#assert os.path.exists(dir_offset), 'Directory "%s" DOES NOT EXIST' % dir_offset
#assert os.path.exists(dir_peds), 'Directory "%s" DOES NOT EXIST' % dir_peds
create_directory(dir_panel, mode=dirmode)
create_directory(dir_peds, mode=dirmode)
create_directory(dir_offset, mode=dirmode)
create_directory(dir_gain, mode=dirmode)
create_directory(dir_rms, mode=dirmode)
create_directory(dir_status, mode=dirmode)
#dark=block[:nrec,:].mean(0) #Calculate mean
#block.sahpe = (1024, 16, 352, 384)
dark, rms, status = proc_dark_block(block[:nrec,idx,:], **opts) # process pedestals per-panel (352, 384)
#continue # TEST
#==========
fname = '%s_pedestals_%s.dat' % (prefix_peds, mode)
save_2darray_in_textfile(dark, fname, filemode, fmt_peds)
fname = '%s_rms_%s.dat' % (prefix_rms, mode)
save_2darray_in_textfile(rms, fname, filemode, fmt_rms)
fname = '%s_status_%s.dat' % (prefix_status, mode)
save_2darray_in_textfile(status, fname, filemode, fmt_status)
#if this is an auto gain ranging mode, also calculate the corresponding _L pedestal:
if mode=='AHL-H': # evaluate AHL_L from AHL_H
ped_hl_h = dark #[3,:,:]
offset_hl_h = load_panel_constants(dir_offset, 'offset_AHL-H', tstamp)
offset_hl_l = load_panel_constants(dir_offset, 'offset_AHL-L', tstamp)
gain_hl_h = load_panel_constants(dir_gain, 'gainci_AHL-H', tstamp)
gain_hl_l = load_panel_constants(dir_gain, 'gainci_AHL-L', tstamp)
#if offset is not None:
if all([v is not None for v in (offset_hl_h, offset_hl_l, gain_hl_h, gain_hl_l)]):
ped_hl_l = offset_hl_l - (offset_hl_h - ped_hl_h) * divide_protected(gain_hl_l, gain_hl_h) #V3 Gabriel's
fname = '%s_pedestals_AHL-L.dat' % prefix_peds
save_2darray_in_textfile(ped_hl_l, fname, filemode, fmt_peds)
elif mode=='AML-M': # evaluate AML_L from AML_M
ped_ml_m = dark #[4,:,:]
offset_ml_m = load_panel_constants(dir_offset, 'offset_AML-M', tstamp)
offset_ml_l = load_panel_constants(dir_offset, 'offset_AML-L', tstamp)
gain_ml_m = load_panel_constants(dir_gain, 'gainci_AML-M', tstamp)
gain_ml_l = load_panel_constants(dir_gain, 'gainci_AML-L', tstamp)
#if offset is not None:
if all([v is not None for v in (offset_ml_m, offset_ml_l, gain_ml_m, gain_ml_l)]):
ped_ml_l = offset_ml_l - (offset_ml_m - ped_ml_m) * divide_protected(gain_ml_l, gain_ml_m) #V3 Gabriel's
fname = '%s_pedestals_AML-L.dat' % prefix_peds
save_2darray_in_textfile(ped_ml_l, fname, filemode, fmt_peds)
def proc_dark_block(block, **opts):
"""Returns per-panel (352, 384) arrays of mean, rms, ...
block.shape = (nrecs, 352, 384), where nrecs <= 1024
"""
exp = opts.get('exp', None)
detname = opts.get('det', None)
int_lo = opts.get('int_lo', 1) # lowest intensity accepted for dark evaluation
int_hi = opts.get('int_hi', 16000) # highest intensity accepted for dark evaluation
intnlo = opts.get('intnlo', 6.0) # intensity ditribution number-of-sigmas low
intnhi = opts.get('intnhi', 6.0) # intensity ditribution number-of-sigmas high
rms_lo = opts.get('rms_lo', 0.001) # rms ditribution low
rms_hi = opts.get('rms_hi', 16000) # rms ditribution high
rmsnlo = opts.get('rmsnlo', 6.0) # rms ditribution number-of-sigmas low
rmsnhi = opts.get('rmsnhi', 6.0) # rms ditribution number-of-sigmas high
fraclm = opts.get('fraclm', 0.1) # allowed fraction limit
fraclo = opts.get('fraclo', 0.05) # fraction of statistics below low gate limit
frachi = opts.get('frachi', 0.95) # fraction of statistics below high gate limit
frac05 = 0.5
nrecs1 = opts.get('nrecs1', None) # number of records for the 1st stage processing
logger.debug('in proc_dark_block for exp=%s det=%s, block.shape=%s' % (exp, detname, str(block.shape)))
logger.info(info_ndarr(block, 'Begin pricessing of the data block:\n ', first=100, last=105))
logger.debug('fraction of statistics for gate limits low: %.3f high: %.3f' % (fraclo, frachi))
t0_sec = time()
nrecs, ny, nx = block.shape
shape = (ny, nx)
if nrecs1 is None or nrecs1>nrecs: nrecs1 = nrecs
arr1_u16 = np.ones(shape, dtype=np.uint16)
arr1 = np.ones(shape, dtype=np.uint64)
t1_sec = time()
"""
NOTE:
- our data is uint16.
- np.median(block, axis=0) or np.quantile(...,interpolation='linear') return result rounded to int
- in order to return interpolated float values apply the trick:
data_block + random [0,1)-0.5
- this would distort data in the range [-0.5,+0.5) ADU, but would allow to get better interpolation for median and quantile values
- use nrecs1 (< nrecs) due to memory and time consumption
"""
#blockf64 = np.random.random(block.shape) - 0.5 + block
blockf64 = np.random.random((nrecs1, ny, nx)) - 0.5 + block[:nrecs1,:]
logger.debug(info_ndarr(blockf64, '1-st stage conversion uint16 to float64, add random [0,1)-0.5 time = %.3f sec '%(time()-t1_sec), first=100, last=105))
t1_sec = time()
#arr_med = np.median(block, axis=0)
arr_med = np.quantile(blockf64, frac05, axis=0, interpolation='linear')
arr_qlo = np.quantile(blockf64, fraclo, axis=0, interpolation='linear')
arr_qhi = np.quantile(blockf64, frachi, axis=0, interpolation='linear')
logger.debug('block array median/quantile(0.5) for med, qlo, qhi time = %.3f sec' % (time()-t1_sec))
med_med = np.median(arr_med)
med_qlo = np.median(arr_qlo)
med_qhi = np.median(arr_qhi)
arr_dev_3d = block[:,] - arr_med # .astype(dtype=np.float64)
arr_abs_dev = np.median(np.abs(arr_dev_3d), axis=0)
med_abs_dev = np.median(arr_abs_dev)
logger.info(info_ndarr(arr_med, ' arr_med[100:105] ', first=100, last=105))
logger.info(info_ndarr(arr_qlo, ' arr_qlo[100:105] ', first=100, last=105))
logger.info(info_ndarr(arr_qhi, ' arr_qhi[100:105] ', first=100, last=105))
logger.info(info_ndarr(arr_abs_dev, ' abs_dev[100:105] ', first=100, last=105))
s = 'Pre-processing time %.3f sec' % (time()-t0_sec)\
+ '\nResults for median over pixels intensities:'\
+ '\n %.3f fraction of the event spectrum is below %.3f ADU - pedestal estimator' % (frac05, med_med)\
+ '\n %.3f fraction of the event spectrum is below %.3f ADU - gate low limit' % (fraclo, med_qlo)\
+ '\n %.3f fraction of the event spectrum is below %.3f ADU - gate upper limit' % (frachi, med_qhi)\
+ '\n event spectrum spread median(abs(raw-med)): %.3f ADU - spectral peak width estimator' % med_abs_dev
logger.info(s)
#sys.exit('TEST EXIT')
logger.debug(info_ndarr(arr_med, '1st iteration proc time = %.3f sec arr_av1' % (time()-t0_sec)))
#gate_half = nsigma*rms_ave
#logger.debug('set gate_half=%.3f for intensity gated average, which is %.3f * sigma' % (gate_half,nsigma))
#gate_half = nsigma*abs_dev_med
#logger.debug('set gate_half=%.3f for intensity gated average, which is %.3f * abs_dev_med' % (gate_half,nsigma))
# 2nd loop over recs in block to evaluate gated parameters
logger.debug('Begin 2nd iteration')
sta_int_lo = np.zeros(shape, dtype=np.uint64)
sta_int_hi = np.zeros(shape, dtype=np.uint64)
arr_max = np.zeros(shape, dtype=block.dtype)
arr_min = np.ones (shape, dtype=block.dtype) * 0x3fff
gate_lo = arr1_u16 * int_lo
gate_hi = arr1_u16 * int_hi
#gate_hi = np.minimum(arr_av1 + gate_half, gate_hi).astype(dtype=block.dtype)
#gate_lo = np.maximum(arr_av1 - gate_half, gate_lo).astype(dtype=block.dtype)
gate_lo = np.maximum(arr_qlo, gate_lo).astype(dtype=block.dtype)
gate_hi = np.minimum(arr_qhi, gate_hi).astype(dtype=block.dtype)
cond = gate_hi>gate_lo
gate_hi[np.logical_not(cond)] +=1
#gate_hi = np.select((cond, np.logical_not(cond)), (gate_hi, gate_hi+1), 0)
logger.debug(info_ndarr(gate_lo, ' gate_lo '))
logger.debug(info_ndarr(gate_hi, ' gate_hi '))
arr_sum0 = np.zeros(shape, dtype=np.uint64)
arr_sum1 = np.zeros(shape, dtype=np.float64)
arr_sum2 = np.zeros(shape, dtype=np.float64)
#blockdbl = np.array(block, dtype=np.float64)
for nrec in range(nrecs):
raw = block[nrec,:]
rawdbl = raw.astype(dtype=np.uint64) # blockdbl[nrec,:]
logger.debug('nrec:%03d median(raw-ave): %f' % (nrec, np.median(raw.astype(dtype=np.float64) - arr_med)))
#logger.debug('nrec:%03d median(raw-ave): %.6f' % (nrec, np.median(raw.astype(dtype=np.float64) - arr_med)))
#logger.debug(info_ndarr(raw, ' raw '))
#logger.debug(info_ndarr(arr_med, ' arr_med '))
condlist = (np.logical_not(np.logical_or(raw<gate_lo, raw>gate_hi)),)
arr_sum0 += np.select(condlist, (arr1,), 0)
arr_sum1 += np.select(condlist, (rawdbl,), 0)
arr_sum2 += np.select(condlist, (np.square(rawdbl),), 0)
sta_int_lo += np.select((raw<int_lo,), (arr1,), 0)
sta_int_hi += np.select((raw>int_hi,), (arr1,), 0)
arr_max = np.maximum(arr_max, raw)
arr_min = np.minimum(arr_min, raw)
arr_av1 = divide_protected(arr_sum1, arr_sum0)
arr_av2 = divide_protected(arr_sum2, arr_sum0)
frac_int_lo = np.array(sta_int_lo/nrecs, dtype=np.float32)
frac_int_hi = np.array(sta_int_hi/nrecs, dtype=np.float32)
arr_rms = np.sqrt(arr_av2 - np.square(arr_av1))
#rms_ave = arr_rms.mean()
rms_ave = mean_constrained(arr_rms, rms_lo, rms_hi)
rms_min, rms_max = evaluate_limits(arr_rms, rmsnlo, rmsnhi, rms_lo, rms_hi, cmt='RMS')
ave_min, ave_max = evaluate_limits(arr_av1, intnlo, intnhi, int_lo, int_hi, cmt='AVE')
arr_sta_rms_hi = np.select((arr_rms>rms_max,), (arr1,), 0)
arr_sta_rms_lo = np.select((arr_rms<rms_min,), (arr1,), 0)
arr_sta_int_hi = np.select((frac_int_hi>fraclm,), (arr1,), 0)
arr_sta_int_lo = np.select((frac_int_lo>fraclm,), (arr1,), 0)
arr_sta_ave_hi = np.select((arr_av1>ave_max,), (arr1,), 0)
arr_sta_ave_lo = np.select((arr_av1<ave_min,), (arr1,), 0)
logger.info('Bad pixel status:'\
+'\n status 1: %8d pixel rms > %.3f' % (arr_sta_rms_hi.sum(), rms_max)\
+'\n status 2: %8d pixel rms < %.3f' % (arr_sta_rms_lo.sum(), rms_min)\
+'\n status 4: %8d pixel intensity > %g in more than %g fraction of events' % (arr_sta_int_hi.sum(), int_hi, fraclm)\
+'\n status 8: %8d pixel intensity < %g in more than %g fraction of events' % (arr_sta_int_lo.sum(), int_lo, fraclm)\
+'\n status 16: %8d pixel average > %g' % (arr_sta_ave_hi.sum(), ave_max)\
+'\n status 32: %8d pixel average < %g' % (arr_sta_ave_lo.sum(), ave_min)\
)
#0/1/2/4/8/16/32 for good/hot-rms/saturated/cold/cold-rms/average above limit/average below limit,
arr_sta = np.zeros(shape, dtype=np.uint64)
arr_sta += arr_sta_rms_hi # hot rms
arr_sta += arr_sta_rms_lo*2 # cold rms
arr_sta += arr_sta_int_hi*4 # satturated
arr_sta += arr_sta_int_lo*8 # cold
arr_sta += arr_sta_ave_hi*16 # too large average
arr_sta += arr_sta_ave_lo*32 # too small average
absdiff_av1_med = np.abs(arr_av1-arr_med)
logger.debug(info_ndarr(absdiff_av1_med, 'np.abs(arr_av1-arr_med)', first=100, last=105))
logger.info('estimator of difference between gated average and median np.median(np.abs(arr_av1-arr_med)): %.3f' % np.median(absdiff_av1_med))
cond = absdiff_av1_med > med_abs_dev
arr_av1[cond] = arr_med[cond]
arr_sta_bad = np.select((cond,), (arr1,), 0)
frac_bad = arr_sta_bad.sum()/float(arr_av1.size)
logger.debug('fraction of panel pixels with gated average deviated from and replaced by median: %.6f' % frac_bad)
logger.info('data block processing time = %.3f sec' % (time()-t0_sec))
logger.debug(info_ndarr(arr_av1, 'arr_av1 [100:105] ', first=100, last=105))
logger.debug(info_ndarr(arr_rms, 'pixel_rms [100:105] ', first=100, last=105))
logger.debug(info_ndarr(arr_sta, 'pixel_status[100:105] ', first=100, last=105))
logger.debug(info_ndarr(arr_med, 'arr mediane [100:105] ', first=100, last=105))
return arr_av1, arr_rms, arr_sta
if nevt > args.events:
print('break at nevt %d' % nevt)
break
if nevt < args.evskip:
print('skip nevt %d' % nevt)
continue
if tname in ('4', '7', '8', '22', '23'):
evt_peds = peds[args.grindex,:] if args.grindex is not None else\
event_constants(det.raw, evt, peds) #(7, 4, 352, 384) -> (4, 352, 384)
print(info_ndarr(evt_peds, 'evt_peds'))
if tname in ('8', '11', '22', '23'):
gfac = divide_protected(np.ones_like(gain), gain)
evt_gfac = gfac[args.grindex,:] if args.grindex is not None else\
event_constants(det.raw, evt, gfac) #(7, 4, 352, 384) -> (4, 352, 384)
print(info_ndarr(evt_gfac, 'evt_gfac, keV/ADU'))
step_evt = 's%02d-e%04d' % (nstep, nevt)
if tname == '1':
suffix = 'segment-nums'
ones = np.ones(det.raw._seg_geo.shape()) # (352,384)
seginds = det.raw._segment_indices() #_segments(evt)
print('seginds', seginds)
arr = np.stack([ones * i for i in seginds])
AMIN, AMAX = amin_amax(args, amin_def=-1, amax_def=4)
elif tname == '2':