def deploy_calib_array(cdir,
src,
type,
run_start,
run_end=None,
arr=None,
dcmts={},
fmt='%.1f',
pbits=1):
"""Deploys array in calibration file
- makes the new file name using make_calib_file_name(...)
- if file with this name already exists - rename it with current timestamp in the name
- save array in file
- add history record
"""
fname = make_calib_file_name(cdir, src, type, run_start, run_end, pbits)
path_history = '%s/HISTORY' % os.path.dirname(fname)
if os.path.exists(fname):
fname_bkp = '%s-%s' % (fname, gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S'))
os.system('cp %s %s' % (fname, fname_bkp))
if pbits & 1:
print('Existing file %s\nis backed-up %s' % (fname, fname_bkp))
# extend dictionary for other parameters
d = dict(dcmts)
d['run'] = run_start
d['fname'] = os.path.basename(fname)
d['src'] = src
d['ctype'] = type
# make list of comments
cmts = ['%s %s' % (k.upper().ljust(11), v) for k, v in d.iteritems()]
# save n-dimensional numpy array in the tmp text file
fntmp = tempfile.NamedTemporaryFile(mode='r+b', suffix='.data')
if pbits & 2: print('Save constants in tmp file: %s' % fntmp.name)
save_txt(fntmp.name, arr, cmts, fmt='%.1f')
if pbits & 1: print('Deploy constants in file: %s' % fname)
# USE cat in stead of cp and move in order to create output file with correct ACL permissions
cmd_cat = 'cat %s > %s' % (fntmp.name, fname)
#os.system(cmd_cat)
stream = os.popen(cmd_cat)
resp = stream.read()
msg = 'Command: %s\n - resp: %s' % (cmd_cat, resp)
if pbits & 2: print(msg)
# add record to the HISTORY file
hrec = _history_record(d)
if pbits & 1: print('Add record: %sto the file: %s' % (hrec, path_history))
gu.save_textfile(hrec, path_history, mode='a')
def save_constants_in_file(tname, fname='0-end.txt'):
if tname == '0':
nda, ctype = delayed_denom()
print_ndarr(nda, 'nda for ctype: %s' % ctype)
save_txt(fname, nda, cmts=(), fmt='%d', verbos=True, addmetad=True)
cmd = 'cdb add -e tmo -d tmott -c %s -r 1 -f %s -u dubrovin' % (ctype,
fname)
elif tname == '1':
d, ctype = fir_coefficients()
print('dict:', d)
save_textfile(str(d), fname, mode='w', verb=True)
cmd = 'cdb add -e tmo -d tmott -c %s -r 1 -f %s -i txt -u dubrovin' % (
ctype, fname)
msg = 'DO NOT FORGET ADD CONSTS TO DB: %s' % cmd
print(msg)
def save_data_in_file(data,
prefix,
control={
'txt': True,
'npy': True
},
fmt='%.3f'):
#elif data_type == 'any':
# gu.save_textfile(str(data), fname, mode='w', verb=verb)
if isinstance(data, np.ndarray):
if control['txt']:
from psana.pscalib.calib.NDArrIO import save_txt # load_txt
fname = '%s.txt' % prefix
save_txt(fname, data, cmts=(), fmt=fmt)
if control['npy']:
fname = '%s.npy' % prefix
np.save(fname, data, allow_pickle=False)
logger.info('saved file: %s' % fname)
else:
logger.warning('DO NOT SAVE unexpected data type: %s' % type(data))
def save_doc(self):
"""Saves document metadata and data in files
"""
logger.debug('In save_doc')
wdoce = cp.cmwdbdoceditor
if wdoce is None or wdoce.data_nda is None:
logger.warning(
'Document editor is not available. Select collection in DB then document in List mode.'
)
return
dicdoc = wdoce.get_model_dicdoc(discard_id_ts=False)
nda = wdoce.get_data_nda()
msg = '\n '.join(
['%12s : %s' % (k, dicdoc[k]) for k in sorted(dicdoc.keys())])
logger.info('Save array and document metadata in files\n%s \n%s' %
(msg, info_ndarr(nda, 'data n-d array ')))
prefix = self.out_fname_prefix(dicdoc)
d = {'data': True, 'meta': True}
resp = change_check_box_dict_in_popup_menu(d, 'Select and confirm',\
msg='Save current document in file\n%s\nfor types:'%prefix, parent=self.but_add)
if resp == 1:
logger.info(
'Save document data and metadata in files with prefix: %s' %
prefix)
for k in d.keys():
if d[k]:
fname = '%s.%s' % (prefix, k)
logger.debug('save_doc %s in file %s' % (k, fname))
if k == 'data': save_txt(fname, nda, cmts=(), fmt='%.1f')
if k == 'meta': save_textfile(msg, fname, mode='w')
else:
logger.warning('Command "Save" is cancelled')
def get(self):
"""Saves in file calibration constants from database.
"""
mode, kwargs = self.mode, self.kwargs
defs = self.defs
host = kwargs.get('host', None)
port = kwargs.get('port', None)
exp = kwargs.get('experiment', None)
det = kwargs.get('detector', None)
ctype = kwargs.get('ctype', None)
run = kwargs.get('run', None)
run_end = kwargs.get('run_end', None)
tsec = kwargs.get('time_sec', None)
tstamp = kwargs.get('time_stamp', None)
vers = kwargs.get('version', None)
fname = kwargs.get('iofname', None)
query = {'detector': det, 'ctype': ctype}
if run != defs['run']:
query['run'] = {'$lte': run}
query['run_end'] = {'$gte': run}
if tsec != defs['time_sec']: query['time_sec'] = {'$lte': tsec}
if vers != defs['version']: query['version'] = vers
#logger.debug('query: %s' % str(query))
dbname = dbu.get_dbname(**kwargs)
client = self.client()
if not self.check_database(client, dbname): return
detname = kwargs.get('detector', None)
if detname is None:
logger.warning(
'%s needs in the collection name. Please specify the detector name.'
% (mode))
colname = detname
db, fs = dbu.db_and_fs(client, dbname)
colnames = dbu.collection_names(db)
if not (colname in colnames): # dbu.collection_exists(db, colname)
logger.warning('db "%s" does not have collection "%s"' %
(db.name, str(colname)))
return
col = dbu.collection(db, colname)
logger.info('Search document in db "%s" collection "%s"' %
(dbname, colname))
#client, expname, detname, db_exp, db_det, fs_exp, fs_det, col_exp, col_det =\
# dbu.connect(host=host, port=port, experiment=exp, detector=det, verbose=verb)
#fs, doc = fs_exp, dbu.find_doc(col, query)
#if doc is None :
# fs, doc = fs_det, dbu.find_doc(col_det, query)
# if doc is None :
# logger.warning('Can not find document for query: %s' % str(query))
# return
doc = dbu.find_doc(col, query)
if doc is None:
logger.warning('Can not find document for query: %s' % str(query))
return
data = dbu.get_data_for_doc(fs, doc)
if data is None:
logger.warning('Can not load data for doc: %s' % str(doc))
return
if fname is None: fname = 'clb-%s-%s-%s.npy' % (expname, det, ctype)
if ctype == 'geometry':
gu.save_textfile(data, fname, mode='w', verb=verb)
elif os.path.splitext(fname)[1] == '.npy':
np.save(fname, data, allow_pickle=False)
else:
save_txt(fname, data, fmt='%.2f')
if verb: logger.info(info_ndarr(data, 'nda', first=0, last=3))
logger.info('Save constants in file: %s' % fname)
def summary(self, evt):
logger.info('%s\nRaw data for %s found/selected in %d events' %
(80 * '_', self.detname, self.counter))
if self.counter:
logger.info(', begin data summary stage')
else:
logger.info(', STOP processing, there are no arrays to save...')
return
t0_sec = time()
# make shorter references
det = self.det
ofname = self.ofname
plotim = self.plotim
savebw = self.savebw
int_hi = self.int_hi
int_lo = self.int_lo
fraclm = self.fraclm
counter = self.counter
arr_av1 = divide_protected(self.arr_sum1, self.arr_sum0)
arr_av2 = divide_protected(self.arr_sum2, self.arr_sum0)
frac_int_lo = np.array(self.sta_int_lo / counter, dtype=np.float32)
frac_int_hi = np.array(self.sta_int_hi / counter, dtype=np.float32)
arr_rms = np.sqrt(arr_av2 - np.square(arr_av1))
rms_min, rms_max = evaluate_limits(arr_rms,
self.rmsnlo,
self.rmsnhi,
self.rms_lo,
self.rms_hi,
cmt='RMS')
ave_min, ave_max = evaluate_limits(arr_av1,
self.intnlo,
self.intnhi,
self.int_lo,
self.int_hi,
cmt='AVE')
arr_sta_rms_hi = np.select((arr_rms > rms_max, ), (self.arr1, ), 0)
arr_sta_rms_lo = np.select((arr_rms < rms_min, ), (self.arr1, ), 0)
arr_sta_int_hi = np.select((frac_int_hi > fraclm, ), (self.arr1, ), 0)
arr_sta_int_lo = np.select((frac_int_lo > fraclm, ), (self.arr1, ), 0)
arr_sta_ave_hi = np.select((arr_av1 > ave_max, ), (self.arr1, ), 0)
arr_sta_ave_lo = np.select((arr_av1 < ave_min, ), (self.arr1, ), 0)
logger.info(' Bad pixel status:')
logger.info(' status 1: %8d pixel rms > %.3f' %
(arr_sta_rms_hi.sum(), rms_max))
logger.info(' status 8: %8d pixel rms < %.3f' %
(arr_sta_rms_lo.sum(), rms_min))
logger.info(
' status 2: %8d pixel intensity > %g in more than %g fraction of events'
% (arr_sta_int_hi.sum(), int_hi, fraclm))
logger.info(
' status 4: %8d pixel intensity < %g in more than %g fraction of events'
% (arr_sta_int_lo.sum(), int_lo, fraclm))
logger.info(' status 16: %8d pixel average > %g' %
(arr_sta_ave_hi.sum(), ave_max))
logger.info(' status 32: %8d pixel average < %g' %
(arr_sta_ave_lo.sum(), ave_min))
#0/1/2/4/8/16/32 for good/hot-rms/saturated/cold/cold-rms/average above limit/average below limit,
arr_sta = np.zeros(self.shape, dtype=np.int64)
arr_sta += arr_sta_rms_hi # hot rms
arr_sta += arr_sta_rms_lo * 8 # cold rms
arr_sta += arr_sta_int_hi * 2 # satturated
arr_sta += arr_sta_int_lo * 4 # cold
arr_sta += arr_sta_ave_hi * 16 # too large average
arr_sta += arr_sta_ave_lo * 32 # too small average
arr_msk = np.select((arr_sta > 0, ), (self.arr0, ), 1)
cmod = None # self._common_mode_pars(arr_av1, arr_rms, arr_msk)
if plotim & 1: plot_det_image(det, evt, arr_av1, tit='average')
if plotim & 2: plot_det_image(det, evt, arr_rms, tit='RMS')
if plotim & 4: plot_det_image(det, evt, arr_sta, tit='status')
if plotim & 8: plot_det_image(det, evt, arr_msk, tit='mask')
if plotim & 16: plot_det_image(det, evt, self.arr_max, tit='maximum')
if plotim & 32: plot_det_image(det, evt, self.arr_min, tit='minimum')
if plotim & 64:
plot_det_image(det,
evt,
self.sta_int_lo,
tit='statistics below threshold')
if plotim & 128:
plot_det_image(det,
evt,
self.sta_int_hi,
tit='statistics above threshold')
cmts = ['DATASET %s' % self.detname, 'STATISTICS %d' % counter]
# Save n-d array in text file %
template = fname_template(self.orun, self.detname, ofname, counter,
self.ev1_sec, self.ev1_nsec)
addmetad = True
verbos = self.verbos
if savebw & 1:
save_txt(template % 'ave', arr_av1, cmts + ['ARR_TYPE average'],
'%8.2f', verbos, addmetad)
if savebw & 2:
save_txt(template % 'rms', arr_rms, cmts + ['ARR_TYPE RMS'],
'%8.2f', verbos, addmetad)
if savebw & 4:
save_txt(template % 'sta', arr_sta, cmts + ['ARR_TYPE status'],
'%d', verbos, addmetad)
if savebw & 8:
save_txt(template % 'msk', arr_msk, cmts + ['ARR_TYPE mask'],
'%1d', verbos, addmetad)
if savebw & 16:
save_txt(template % 'max', self.arr_max, cmts + ['ARR_TYPE max'],
'%d', verbos, addmetad)
if savebw & 32:
save_txt(template % 'min', self.arr_min, cmts + ['ARR_TYPE min'],
'%d', verbos, addmetad)
if savebw & 64 and cmod is not None:
np.savetxt(template % 'cmo',
cmod,
fmt='%d',
delimiter=' ',
newline=' ')
save_txt(template % 'cmm', cmod, cmts + ['ARR_TYPE common_mode'],
'%d', verbos, False)
args = vars(self.args)
version = args.get('version', str_tstamp('V%Y%m%dT%H%M%S', t0_sec))
orun = self.orun
loglev = args.get('loglev', 'INFO')
# args.expnam # args.runnum, orun.timestamp
kwa = {
'detname': self.detname,
'experiment': orun.expt,
'runnum': orun.runnum,
'timestamp': str_tstamp(time_sec=self.ev1_sec),
'version': version,
'comment': str(cmts),
'loglev': loglev,
}
logger.debug('kwa:', kwa)
logger.debug('args:', self.args)
if self.addcdb:
from psana.pscalib.calib.MDBCommands import add_constants
if savebw & 1: add_constants(template % 'ave', 'pedestals', **kwa)
if savebw & 2: add_constants(template % 'rms', 'pixel_rms', **kwa)
if savebw & 4:
add_constants(template % 'sta', 'pixel_status', **kwa)
if savebw & 8: add_constants(template % 'msk', 'pixel_mask', **kwa)
if savebw & 16: add_constants(template % 'max', 'pixel_max', **kwa)
if savebw & 32: add_constants(template % 'min', 'pixel_min', **kwa)
logger.info('Data summary for %s is completed, dt=%7.3f sec' %
(self.detname, time() - t0_sec))
def save_ndarray_in_textfile(nda, fname, fmode, fmt):
fexists = os.path.exists(fname)
save_txt(fname=fname, arr=nda, fmt=fmt)
if not fexists: set_file_access_mode(fname, fmode)
logger.debug('saved: %s fmode: %s fmt: %s' % (fname, oct(fmode), fmt))
def summary(self, evt):
logger.info('%s\nRaw data for %s found/selected in %d events' %
(80 * '_', self.detname, self.counter))
if self.counter:
logger.info(', begin data summary stage')
else:
logger.info(', STOP processing, there are no arrays to save...')
return
t0_sec = time()
# make shorter references
det = self.det
ofname = self.ofname
plotim = self.plotim
savebw = self.savebw
verbos = self.verbos
int_hi = self.int_hi
int_lo = self.int_lo
fraclm = self.fraclm
counter = self.counter
arr_av1 = divide_protected(self.arr_sum1, self.arr_sum0)
arr_av2 = divide_protected(self.arr_sum2, self.arr_sum0)
frac_int_lo = np.array(self.sta_int_lo / counter, dtype=np.float32)
frac_int_hi = np.array(self.sta_int_hi / counter, dtype=np.float32)
arr_rms = np.sqrt(arr_av2 - np.square(arr_av1))
rms_min, rms_max = evaluate_limits(arr_rms,
self.rmsnlo,
self.rmsnhi,
self.rms_lo,
self.rms_hi,
self.verbos,
cmt='RMS')
#if self.rms_lo is None or self.rms_hi is None or self.rms_hi == 0.\
#else (self.rms_lo, self.rms_hi)
#if rms_min<0 : rms_min=0
ave_min, ave_max = evaluate_limits(arr_av1, self.intnlo, self.intnhi, self.int_lo, self.int_hi, self.verbos, cmt='AVE')\
arr_sta_rms_hi = np.select((arr_rms > rms_max, ), (self.arr1, ), 0)
arr_sta_rms_lo = np.select((arr_rms < rms_min, ), (self.arr1, ), 0)
arr_sta_int_hi = np.select((frac_int_hi > fraclm, ), (self.arr1, ), 0)
arr_sta_int_lo = np.select((frac_int_lo > fraclm, ), (self.arr1, ), 0)
arr_sta_ave_hi = np.select((arr_av1 > ave_max, ), (self.arr1, ), 0)
arr_sta_ave_lo = np.select((arr_av1 < ave_min, ), (self.arr1, ), 0)
logger.info(' Bad pixel status:')
logger.info(' status 1: %8d pixel rms > %.3f' %
(arr_sta_rms_hi.sum(), rms_max))
logger.info(' status 8: %8d pixel rms < %.3f' %
(arr_sta_rms_lo.sum(), rms_min))
logger.info(
' status 2: %8d pixel intensity > %g in more than %g fraction of events'
% (arr_sta_int_hi.sum(), int_hi, fraclm))
logger.info(
' status 4: %8d pixel intensity < %g in more than %g fraction of events'
% (arr_sta_int_lo.sum(), int_lo, fraclm))
logger.info(' status 16: %8d pixel average > %g' %
(arr_sta_ave_hi.sum(), ave_max))
logger.info(' status 32: %8d pixel average < %g' %
(arr_sta_ave_lo.sum(), ave_min))
#0/1/2/4/8/16/32 for good/hot-rms/saturated/cold/cold-rms/average above limit/average below limit,
arr_sta = np.zeros(self.shape, dtype=np.int64)
arr_sta += arr_sta_rms_hi # hot rms
arr_sta += arr_sta_rms_lo * 8 # cold rms
arr_sta += arr_sta_int_hi * 2 # satturated
arr_sta += arr_sta_int_lo * 4 # cold
arr_sta += arr_sta_ave_hi * 16 # too large average
arr_sta += arr_sta_ave_lo * 32 # too small average
arr_msk = np.select((arr_sta > 0, ), (self.arr0, ), 1)
cmod = None # self._common_mode_pars(arr_av1, arr_rms, arr_msk)
if plotim & 1: plot_det_image(det, evt, arr_av1, tit='average')
if plotim & 2: plot_det_image(det, evt, arr_rms, tit='RMS')
if plotim & 4: plot_det_image(det, evt, arr_sta, tit='status')
if plotim & 8: plot_det_image(det, evt, arr_msk, tit='mask')
if plotim & 16: plot_det_image(det, evt, self.arr_max, tit='maximum')
if plotim & 32: plot_det_image(det, evt, self.arr_min, tit='minimum')
if plotim & 64:
plot_det_image(det,
evt,
self.sta_int_lo,
tit='statistics below threshold')
if plotim & 128:
plot_det_image(det,
evt,
self.sta_int_hi,
tit='statistics above threshold')
cmts = ['DATASET %s' % self.detname, 'STATISTICS %d' % counter]
# Save n-d array in text file %
template = fname_template(self.orun, self.detname, ofname, counter,
self.ev1_sec, self.ev1_nsec)
addmetad = True
if savebw & 1:
save_txt(template % 'ave', arr_av1, cmts + ['ARR_TYPE average'],
'%8.2f', verbos, addmetad)
if savebw & 2:
save_txt(template % 'rms', arr_rms, cmts + ['ARR_TYPE RMS'],
'%8.2f', verbos, addmetad)
if savebw & 4:
save_txt(template % 'sta', arr_sta, cmts + ['ARR_TYPE status'],
'%d', verbos, addmetad)
if savebw & 8:
save_txt(template % 'msk', arr_msk, cmts + ['ARR_TYPE mask'],
'%1d', verbos, addmetad)
if savebw & 16:
save_txt(template % 'max', self.arr_max, cmts + ['ARR_TYPE max'],
'%d', verbos, addmetad)
if savebw & 32:
save_txt(template % 'min', self.arr_min, cmts + ['ARR_TYPE min'],
'%d', verbos, addmetad)
if savebw & 64 and cmod is not None:
np.savetxt(template % 'cmo',
cmod,
fmt='%d',
delimiter=' ',
newline=' ')
save_txt(template % 'cmm', cmod, cmts + ['ARR_TYPE common_mode'],
'%d', verbos, False)
if self.verbos & 1:
logger.info('Data summary for %s is completed, dt=%7.3f sec' %
(self.detname, time() - t0_sec))