def add(self):
"""Adds calibration constants to database from file.
"""
kwa = self.kwargs
fname = kwa.get('iofname', 'None')
ctype = kwa.get('ctype', 'None')
dtype = kwa.get('dtype', 'None')
verb = self.loglevel == 'DEBUG'
data = mu.data_from_file(fname, ctype, dtype, verb)
mu.insert_calib_data(data, **kwa)
def deploy_constants(*args, **kwa):
from psana.pscalib.calib.NDArrIO import save_txt; global save_txt
import psana.pscalib.calib.MDBUtils as mu
import psana.pscalib.calib.MDBWebUtils as wu
cc = wu.cc # import psana.pscalib.calib.CalibConstants as cc
exp = kwa.get('exp', None)
detname = kwa.get('det', None)
runs = kwa.get('runs', None)
tstamp = kwa.get('tstamp', None) # (int) time stamp in format YYYYmmddHHMMSS or run number(<10000)
dirxtc = kwa.get('dirxtc', None)
dirrepo = kwa.get('dirrepo', CALIB_REPO_EPIX10KA)
deploy = kwa.get('deploy', False)
fmt_peds = kwa.get('fmt_peds', '%.3f')
fmt_gain = kwa.get('fmt_gain', '%.6f')
fmt_rms = kwa.get('fmt_rms', '%.3f')
fmt_status = kwa.get('fmt_status', '%4i')
logmode = kwa.get('logmode', 'DEBUG')
dirmode = kwa.get('dirmode', 0o777)
filemode = kwa.get('filemode', 0o666)
high = kwa.get('high', 16.40) # ADU/keV
medium = kwa.get('medium', 5.466) # ADU/keV
low = kwa.get('low', 0.164) # ADU/keV
proc = kwa.get('proc', 'prsg')
paninds = kwa.get('paninds', None)
version = kwa.get('version', 'N/A')
run_end = kwa.get('run_end', 'end')
comment = kwa.get('comment', 'no comment')
dbsuffix = kwa.get('dbsuffix', '')
logger.setLevel(DICT_NAME_TO_LEVEL[logmode])
#dsname = 'exp=%s:run=%d'%(exp,irun) if dirxtc is None else 'exp=%s:run=%d:dir=%s'%(exp, irun, dirxtc)
irun = irun_first(runs)
#_name = sys._getframe().f_code.co_name
_name = SCRNAME
save_log_record_at_start(dirrepo, _name, dirmode, filemode, logmode)
cpdic = get_config_info_for_dataset_detname(**kwa)
tstamp_run = cpdic.get('tstamp', None) # str
expnum = cpdic.get('expnum', None)
shape = cpdic.get('shape', None)
strsrc = cpdic.get('strsrc', None)
panel_ids = cpdic.get('panel_ids', None)
panel_inds = cpdic.get('panel_inds',None)
dettype = cpdic.get('dettype', None)
det_name = cpdic.get('det_name', None)
longname = cpdic.get('longname', detname)
gains_def = cpdic.get('gains_def', None)
req_inds = None if paninds is None else [int(i) for i in paninds.split(',')] # conv str '0,1,2,3' to list [0,1,2,3]
logger.info('In %s\n detector: "%s" \n requested_inds: %s' % (_name, detname, str(req_inds)))
assert isinstance(gains_def, tuple)
assert len(gains_def) == 3
if high is None: high = gains_def[0]
if medium is None: medium = gains_def[1]
if low is None: low = gains_def[2]
global GAIN_FACTOR_DEF
#GAIN_MODES = ['FH','FM','FL','AHL-H','AML-M','AHL-L','AML-L']
GAIN_FACTOR_DEF = [high, medium, low, high, medium, low, low]
CTYPE_FMT = {'pedestals' : fmt_peds,
'pixel_gain' : fmt_gain,
'pixel_rms' : fmt_rms,
'pixel_status': fmt_status}
CTYPE_DTYPE = cc.dic_calib_name_to_dtype # {'pedestals': np.float32,...}
logger.debug('detector "%s" panel ids:\n %s' % (detname, '\n '.join(panel_ids)))
#if tstamp is None: tstamp = tstamp_run
_tstamp = tstamp_run
logger.debug('search for calibration files with tstamp <= %s' % _tstamp)
# dict_consts for constants octype: 'pixel_gain', 'pedestals', etc.
dic_consts = {}
for ind, panel_id in zip(panel_inds,panel_ids):
if req_inds is not None and not (ind in req_inds): continue # skip non-selected panels
logger.info('%s\nmerge constants for panel:%02d id: %s' % (98*'_', ind, panel_id))
dir_panel, dir_offset, dir_peds, dir_plots, dir_work, dir_gain, dir_rms, dir_status = dir_names(dirrepo, panel_id)
fname_prefix, panel_alias = file_name_prefix(dirrepo, dettype, panel_id, _tstamp, exp, irun)
prefix_offset, prefix_peds, prefix_plots, prefix_gain, prefix_rms, prefix_status =\
path_prefixes(fname_prefix, dir_offset, dir_peds, dir_plots, dir_gain, dir_rms, dir_status)
#mpars = (('pedestals', 'pedestals', prefix_peds, dir_peds),\
# ('rms', 'pixel_rms', prefix_rms, dir_rms),\
# ('status', 'pixel_status', prefix_status, dir_status),\
# ('gain', 'pixel_gain', prefix_gain, dir_gain))
mpars = []
if 'p' in proc: mpars.append(('pedestals', 'pedestals', prefix_peds, dir_peds))
if 'r' in proc: mpars.append(('rms', 'pixel_rms', prefix_rms, dir_rms))
if 's' in proc: mpars.append(('status', 'pixel_status', prefix_status, dir_status))
if 'g' in proc: mpars.append(('gain', 'pixel_gain', prefix_gain, dir_gain))
if 'c' in proc: mpars.append(('gainci', 'pixel_gain', prefix_gain, dir_gain))
if 'c' in proc:
add_links_for_gainci_fixed_modes(dir_gain, fname_prefix) # FH->AHL-H, FM->AML-M, FL->AML-L/AHL-L
for (ctype, octype, prefix, dir_ctype) in mpars:
fmt = CTYPE_FMT.get(octype,'%.5f')
nda_dtype = CTYPE_DTYPE.get(octype, np.float32)
logger.debug('begin merging for ctype:%s, octype:%s, fmt:%s,\n prefix:%s' % (ctype, octype, fmt, prefix))
fname = '%s_%s.txt' % (prefix, ctype)
nda = merge_panel_gain_ranges(dir_ctype, panel_id, ctype, _tstamp, shape, nda_dtype, fname, fmt, filemode)
if octype in dic_consts: dic_consts[octype].append(nda) # append for panel per ctype
else: dic_consts[octype] = [nda,]
logger.info('\n%s\nMERGE PANEL CONSTANTS AND DEPLOY THEM\n' % (80*'_'))
#if deploy:
dmerge = dir_merge(dirrepo)
create_directory(dmerge, mode=dirmode)
fmerge_prefix = fname_prefix_merge(dmerge, detname, _tstamp, exp, irun)
for octype, lst in dic_consts.items():
mrg_nda = merge_panels(lst)
logger.info(info_ndarr(mrg_nda, 'merged constants for %s ' % octype))
fmerge = '%s-%s.txt' % (fmerge_prefix, octype)
fmt = CTYPE_FMT.get(octype,'%.5f')
save_ndarray_in_textfile(mrg_nda, fmerge, filemode, fmt)
if True: # deploy:
# check opt "-t" if constants need to be deployed with diffiernt time stamp or run number
use_external_run = tstamp is not None and tstamp<10000
use_external_ts = tstamp is not None and tstamp>9999
tvalid_sec = time_sec_from_stamp(fmt=cc.TSFORMAT_SHORT, time_stamp=str(tstamp))\
if use_external_ts else cpdic.get('trun_sec', None)
ivalid_run = tstamp if use_external_run else irun\
if not use_external_ts else 0
dtype = 'ndarray'
kwa = {
'iofname' : fmerge,
'experiment' : exp,
'ctype' : octype,
'dtype' : dtype,
'detector' : detname,
'detname' : det_name,
'longname' : longname,
'time_sec' : tvalid_sec,
'time_stamp' : str_tstamp(fmt=cc.TSFORMAT, time_sec=int(tvalid_sec)),
'tsshort' : str_tstamp(fmt=cc.TSFORMAT_SHORT, time_sec=int(tvalid_sec)),
'tstamp_orig': cpdic.get('tsrun_dark', None),
'run' : ivalid_run,
'run_end' : run_end,
'run_orig' : irun,
'version' : version,
'comment' : comment,
'extpars' : {'content':'extended parameters dict->json->str',},
'dettype' : dettype,
'dbsuffix' : dbsuffix
}
logger.debug('DEPLOY metadata: %s' % str(kwa))
data = mu.data_from_file(fmerge, octype, dtype, True)
logger.debug(info_ndarr(data, 'merged constants loaded from file'))
if deploy:
resp = wu.deploy_constants(data, exp, longname, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS, **kwa)
#id_data_exp, id_data_det, id_doc_exp, id_doc_det = resp if resp is not None
else:
logger.warning('TO DEPLOY CONSTANTS ADD OPTION -D')