def tstamps_run_and_now(trun_sec): # unix epoch time, e.g. 1607569818.532117 sec
"""DEPRECATED HERE - USE FROM UtilsCalib.py
Returns (str) tstamp_run, tstamp_now#, e.g. (str) 20201209191018, 20201217140026
"""
ts_run = str_tstamp(fmt='%Y%m%d%H%M%S', time_sec=trun_sec)
ts_now = str_tstamp(fmt='%Y%m%d%H%M%S', time_sec=None)
return ts_run, ts_now
def add_metadata_kwargs(orun, odet, **kwa):
trun_sec = seconds(orun.timestamp) # 1607569818.532117 sec
# check opt "-t" if constants need to be deployed with diffiernt time stamp or run number
tstamp = kwa.get('tstamp', None)
use_external_run = tstamp is not None and tstamp < 10000
use_external_ts = tstamp is not None and tstamp > 9999
tvalid_sec = time_sec_from_stamp(fmt=cc.TSFORMAT_SHORT, time_stamp=str(tstamp))\
if use_external_ts else trun_sec
ivalid_run = tstamp if use_external_run else orun.runnum\
if not use_external_ts else 0
kwa['experiment'] = orun.expt
kwa['detector'] = odet.raw._uniqueid
kwa['detname'] = odet.raw._det_name
kwa['dettype'] = odet.raw._dettype
kwa['longname'] = odet.raw._uniqueid
kwa['time_sec'] = tvalid_sec
kwa['time_stamp'] = str_tstamp(fmt=cc.TSFORMAT, time_sec=int(tvalid_sec))
kwa['tsshort'] = str_tstamp(fmt=cc.TSFORMAT_SHORT,
time_sec=int(tvalid_sec))
kwa['tstamp_orig'] = str_tstamp(fmt=cc.TSFORMAT, time_sec=int(trun_sec))
kwa['run'] = ivalid_run
kwa['run_end'] = kwa.get('run_end', 'end')
kwa['run_orig'] = orun.runnum
kwa['version'] = kwa.get('version', 'N/A')
kwa['comment'] = kwa.get('comment', 'no comment')
kwa['extpars'] = {
'content': 'extended parameters dict->json->str',
}
return kwa
def get_config_info_for_dataset_detname(**kwargs):
detname = kwargs.get('det', None)
idx = kwargs.get('idx', None)
ds = DataSource(**data_source_kwargs(**kwargs))
logger.debug('ds.runnum_list = %s' % str(ds.runnum_list))
logger.debug('ds.detectors = %s' % str(ds.detectors))
logger.info('ds.xtc_files:\n %s' % ('\n '.join(ds.xtc_files)))
#for orun in ds.runs():
orun = next(ds.runs())
if orun:
logger.debug('==run.runnum : %d' % orun.runnum) # 27
logger.debug(' run.detnames : %s' % str(orun.detnames)) # {'epixquad'}
logger.debug(' run.expt : %s', orun.expt) # ueddaq02
runtstamp = orun.timestamp # 4193682596073796843 relative to 1990-01-01
trun_sec = seconds(runtstamp) # 1607569818.532117 sec
#tstamp_run = str_tstamp(time_sec=int(trun_sec)) #fmt='%Y-%m-%dT%H:%M:%S%z'
tstamp_run, tstamp_now = tstamps_run_and_now(int(trun_sec)) # (str) 20201209191018, 20201217140026
logger.debug(' run.timestamp: %d' % orun.timestamp)
logger.debug(' run unix epoch time %06f sec' % trun_sec)
logger.debug(' run tstamp: %s' % tstamp_run)
logger.debug(' now tstamp: %s' % tstamp_now)
det = orun.Detector(detname)
#co = det.raw._config_object()
cpdic = {}
cpdic['expname'] = orun.expt
cpdic['strsrc'] = None
cpdic['shape'] = det.raw._seg_geo.shape() # (352, 384) for epix10ka or (288,384) for epixhr2x2
cpdic['gain_mode'] = ue.find_gain_mode(det.raw, evt=None) #data=raw: distinguish 5-modes w/o data
cpdic['panel_ids'] = det.raw._segment_ids() #ue.segment_ids_det(det)
cpdic['panel_inds'] = det.raw._segment_indices() #ue.segment_indices_det(det)
cpdic['longname'] = det.raw._fullname() #ue.fullname_det(det) #det.raw._uniqueid
cpdic['det_name'] = det._det_name # det.raw._det_name epixquad
cpdic['dettype'] = det._dettype # epix
cpdic['tstamp'] = tstamp_run # (str) 20201209191018
cpdic['tstamp_now'] = tstamp_now # (str) 20201217140026
cpdic['trun_sec'] = int(trun_sec) # 1607569818.532117 sec
cpdic['tsrun_dark'] = str_tstamp(time_sec=int(trun_sec)) #fmt='%Y-%m-%dT%H:%M:%S%z'
cpdic['gains_def'] = det.raw._gains_def # e.g. for epix10ka (16.4, 5.466, 0.164) ADU/keV
return cpdic
def deploy_constants(*args, **kwa):
from psana.pscalib.calib.NDArrIO import save_txt; global save_txt
import psana.pscalib.calib.MDBUtils as mu
import psana.pscalib.calib.MDBWebUtils as wu
cc = wu.cc # import psana.pscalib.calib.CalibConstants as cc
exp = kwa.get('exp', None)
detname = kwa.get('det', None)
runs = kwa.get('runs', None)
tstamp = kwa.get('tstamp', None) # (int) time stamp in format YYYYmmddHHMMSS or run number(<10000)
dirxtc = kwa.get('dirxtc', None)
dirrepo = kwa.get('dirrepo', CALIB_REPO_EPIX10KA)
deploy = kwa.get('deploy', False)
fmt_peds = kwa.get('fmt_peds', '%.3f')
fmt_gain = kwa.get('fmt_gain', '%.6f')
fmt_rms = kwa.get('fmt_rms', '%.3f')
fmt_status = kwa.get('fmt_status', '%4i')
logmode = kwa.get('logmode', 'DEBUG')
dirmode = kwa.get('dirmode', 0o777)
filemode = kwa.get('filemode', 0o666)
high = kwa.get('high', 16.40) # ADU/keV
medium = kwa.get('medium', 5.466) # ADU/keV
low = kwa.get('low', 0.164) # ADU/keV
proc = kwa.get('proc', 'prsg')
paninds = kwa.get('paninds', None)
version = kwa.get('version', 'N/A')
run_end = kwa.get('run_end', 'end')
comment = kwa.get('comment', 'no comment')
dbsuffix = kwa.get('dbsuffix', '')
logger.setLevel(DICT_NAME_TO_LEVEL[logmode])
#dsname = 'exp=%s:run=%d'%(exp,irun) if dirxtc is None else 'exp=%s:run=%d:dir=%s'%(exp, irun, dirxtc)
irun = irun_first(runs)
#_name = sys._getframe().f_code.co_name
_name = SCRNAME
save_log_record_at_start(dirrepo, _name, dirmode, filemode, logmode)
cpdic = get_config_info_for_dataset_detname(**kwa)
tstamp_run = cpdic.get('tstamp', None) # str
expnum = cpdic.get('expnum', None)
shape = cpdic.get('shape', None)
strsrc = cpdic.get('strsrc', None)
panel_ids = cpdic.get('panel_ids', None)
panel_inds = cpdic.get('panel_inds',None)
dettype = cpdic.get('dettype', None)
det_name = cpdic.get('det_name', None)
longname = cpdic.get('longname', detname)
gains_def = cpdic.get('gains_def', None)
req_inds = None if paninds is None else [int(i) for i in paninds.split(',')] # conv str '0,1,2,3' to list [0,1,2,3]
logger.info('In %s\n detector: "%s" \n requested_inds: %s' % (_name, detname, str(req_inds)))
assert isinstance(gains_def, tuple)
assert len(gains_def) == 3
if high is None: high = gains_def[0]
if medium is None: medium = gains_def[1]
if low is None: low = gains_def[2]
global GAIN_FACTOR_DEF
#GAIN_MODES = ['FH','FM','FL','AHL-H','AML-M','AHL-L','AML-L']
GAIN_FACTOR_DEF = [high, medium, low, high, medium, low, low]
CTYPE_FMT = {'pedestals' : fmt_peds,
'pixel_gain' : fmt_gain,
'pixel_rms' : fmt_rms,
'pixel_status': fmt_status}
CTYPE_DTYPE = cc.dic_calib_name_to_dtype # {'pedestals': np.float32,...}
logger.debug('detector "%s" panel ids:\n %s' % (detname, '\n '.join(panel_ids)))
#if tstamp is None: tstamp = tstamp_run
_tstamp = tstamp_run
logger.debug('search for calibration files with tstamp <= %s' % _tstamp)
# dict_consts for constants octype: 'pixel_gain', 'pedestals', etc.
dic_consts = {}
for ind, panel_id in zip(panel_inds,panel_ids):
if req_inds is not None and not (ind in req_inds): continue # skip non-selected panels
logger.info('%s\nmerge constants for panel:%02d id: %s' % (98*'_', ind, panel_id))
dir_panel, dir_offset, dir_peds, dir_plots, dir_work, dir_gain, dir_rms, dir_status = dir_names(dirrepo, panel_id)
fname_prefix, panel_alias = file_name_prefix(dirrepo, dettype, panel_id, _tstamp, exp, irun)
prefix_offset, prefix_peds, prefix_plots, prefix_gain, prefix_rms, prefix_status =\
path_prefixes(fname_prefix, dir_offset, dir_peds, dir_plots, dir_gain, dir_rms, dir_status)
#mpars = (('pedestals', 'pedestals', prefix_peds, dir_peds),\
# ('rms', 'pixel_rms', prefix_rms, dir_rms),\
# ('status', 'pixel_status', prefix_status, dir_status),\
# ('gain', 'pixel_gain', prefix_gain, dir_gain))
mpars = []
if 'p' in proc: mpars.append(('pedestals', 'pedestals', prefix_peds, dir_peds))
if 'r' in proc: mpars.append(('rms', 'pixel_rms', prefix_rms, dir_rms))
if 's' in proc: mpars.append(('status', 'pixel_status', prefix_status, dir_status))
if 'g' in proc: mpars.append(('gain', 'pixel_gain', prefix_gain, dir_gain))
if 'c' in proc: mpars.append(('gainci', 'pixel_gain', prefix_gain, dir_gain))
if 'c' in proc:
add_links_for_gainci_fixed_modes(dir_gain, fname_prefix) # FH->AHL-H, FM->AML-M, FL->AML-L/AHL-L
for (ctype, octype, prefix, dir_ctype) in mpars:
fmt = CTYPE_FMT.get(octype,'%.5f')
nda_dtype = CTYPE_DTYPE.get(octype, np.float32)
logger.debug('begin merging for ctype:%s, octype:%s, fmt:%s,\n prefix:%s' % (ctype, octype, fmt, prefix))
fname = '%s_%s.txt' % (prefix, ctype)
nda = merge_panel_gain_ranges(dir_ctype, panel_id, ctype, _tstamp, shape, nda_dtype, fname, fmt, filemode)
if octype in dic_consts: dic_consts[octype].append(nda) # append for panel per ctype
else: dic_consts[octype] = [nda,]
logger.info('\n%s\nMERGE PANEL CONSTANTS AND DEPLOY THEM\n' % (80*'_'))
#if deploy:
dmerge = dir_merge(dirrepo)
create_directory(dmerge, mode=dirmode)
fmerge_prefix = fname_prefix_merge(dmerge, detname, _tstamp, exp, irun)
for octype, lst in dic_consts.items():
mrg_nda = merge_panels(lst)
logger.info(info_ndarr(mrg_nda, 'merged constants for %s ' % octype))
fmerge = '%s-%s.txt' % (fmerge_prefix, octype)
fmt = CTYPE_FMT.get(octype,'%.5f')
save_ndarray_in_textfile(mrg_nda, fmerge, filemode, fmt)
if True: # deploy:
# check opt "-t" if constants need to be deployed with diffiernt time stamp or run number
use_external_run = tstamp is not None and tstamp<10000
use_external_ts = tstamp is not None and tstamp>9999
tvalid_sec = time_sec_from_stamp(fmt=cc.TSFORMAT_SHORT, time_stamp=str(tstamp))\
if use_external_ts else cpdic.get('trun_sec', None)
ivalid_run = tstamp if use_external_run else irun\
if not use_external_ts else 0
dtype = 'ndarray'
kwa = {
'iofname' : fmerge,
'experiment' : exp,
'ctype' : octype,
'dtype' : dtype,
'detector' : detname,
'detname' : det_name,
'longname' : longname,
'time_sec' : tvalid_sec,
'time_stamp' : str_tstamp(fmt=cc.TSFORMAT, time_sec=int(tvalid_sec)),
'tsshort' : str_tstamp(fmt=cc.TSFORMAT_SHORT, time_sec=int(tvalid_sec)),
'tstamp_orig': cpdic.get('tsrun_dark', None),
'run' : ivalid_run,
'run_end' : run_end,
'run_orig' : irun,
'version' : version,
'comment' : comment,
'extpars' : {'content':'extended parameters dict->json->str',},
'dettype' : dettype,
'dbsuffix' : dbsuffix
}
logger.debug('DEPLOY metadata: %s' % str(kwa))
data = mu.data_from_file(fmerge, octype, dtype, True)
logger.debug(info_ndarr(data, 'merged constants loaded from file'))
if deploy:
resp = wu.deploy_constants(data, exp, longname, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS, **kwa)
#id_data_exp, id_data_det, id_doc_exp, id_doc_det = resp if resp is not None
else:
logger.warning('TO DEPLOY CONSTANTS ADD OPTION -D')
def timestamp_run(run, fmt='%Y-%m-%dT%H:%M:%S'):
"""converts LCLS2 run.timestamp to human readable timestamp, e.g. 2020-10-27T12:54:47
"""
return str_tstamp(fmt=fmt, time_sec=seconds(run.timestamp))