def add(self): """Adds calibration constants to database from file. """ kwa = self.kwargs fname = kwa.get('iofname', 'None') ctype = kwa.get('ctype', 'None') dtype = kwa.get('dtype', 'None') verb = self.loglevel == 'DEBUG' data = mu.data_from_file(fname, ctype, dtype, verb) mu.insert_calib_data(data, **kwa)
def deploy_constants(*args, **kwa): from psana.pscalib.calib.NDArrIO import save_txt; global save_txt import psana.pscalib.calib.MDBUtils as mu import psana.pscalib.calib.MDBWebUtils as wu cc = wu.cc # import psana.pscalib.calib.CalibConstants as cc exp = kwa.get('exp', None) detname = kwa.get('det', None) runs = kwa.get('runs', None) tstamp = kwa.get('tstamp', None) # (int) time stamp in format YYYYmmddHHMMSS or run number(<10000) dirxtc = kwa.get('dirxtc', None) dirrepo = kwa.get('dirrepo', CALIB_REPO_EPIX10KA) deploy = kwa.get('deploy', False) fmt_peds = kwa.get('fmt_peds', '%.3f') fmt_gain = kwa.get('fmt_gain', '%.6f') fmt_rms = kwa.get('fmt_rms', '%.3f') fmt_status = kwa.get('fmt_status', '%4i') logmode = kwa.get('logmode', 'DEBUG') dirmode = kwa.get('dirmode', 0o777) filemode = kwa.get('filemode', 0o666) high = kwa.get('high', 16.40) # ADU/keV medium = kwa.get('medium', 5.466) # ADU/keV low = kwa.get('low', 0.164) # ADU/keV proc = kwa.get('proc', 'prsg') paninds = kwa.get('paninds', None) version = kwa.get('version', 'N/A') run_end = kwa.get('run_end', 'end') comment = kwa.get('comment', 'no comment') dbsuffix = kwa.get('dbsuffix', '') logger.setLevel(DICT_NAME_TO_LEVEL[logmode]) #dsname = 'exp=%s:run=%d'%(exp,irun) if dirxtc is None else 'exp=%s:run=%d:dir=%s'%(exp, irun, dirxtc) irun = irun_first(runs) #_name = sys._getframe().f_code.co_name _name = SCRNAME save_log_record_at_start(dirrepo, _name, dirmode, filemode, logmode) cpdic = get_config_info_for_dataset_detname(**kwa) tstamp_run = cpdic.get('tstamp', None) # str expnum = cpdic.get('expnum', None) shape = cpdic.get('shape', None) strsrc = cpdic.get('strsrc', None) panel_ids = cpdic.get('panel_ids', None) panel_inds = cpdic.get('panel_inds',None) dettype = cpdic.get('dettype', None) det_name = cpdic.get('det_name', None) longname = cpdic.get('longname', detname) gains_def = cpdic.get('gains_def', None) req_inds = None if paninds is None else [int(i) for i in paninds.split(',')] # conv str '0,1,2,3' to list [0,1,2,3] logger.info('In %s\n detector: "%s" \n requested_inds: %s' % (_name, detname, str(req_inds))) assert isinstance(gains_def, tuple) assert len(gains_def) == 3 if high is None: high = gains_def[0] if medium is None: medium = gains_def[1] if low is None: low = gains_def[2] global GAIN_FACTOR_DEF #GAIN_MODES = ['FH','FM','FL','AHL-H','AML-M','AHL-L','AML-L'] GAIN_FACTOR_DEF = [high, medium, low, high, medium, low, low] CTYPE_FMT = {'pedestals' : fmt_peds, 'pixel_gain' : fmt_gain, 'pixel_rms' : fmt_rms, 'pixel_status': fmt_status} CTYPE_DTYPE = cc.dic_calib_name_to_dtype # {'pedestals': np.float32,...} logger.debug('detector "%s" panel ids:\n %s' % (detname, '\n '.join(panel_ids))) #if tstamp is None: tstamp = tstamp_run _tstamp = tstamp_run logger.debug('search for calibration files with tstamp <= %s' % _tstamp) # dict_consts for constants octype: 'pixel_gain', 'pedestals', etc. dic_consts = {} for ind, panel_id in zip(panel_inds,panel_ids): if req_inds is not None and not (ind in req_inds): continue # skip non-selected panels logger.info('%s\nmerge constants for panel:%02d id: %s' % (98*'_', ind, panel_id)) dir_panel, dir_offset, dir_peds, dir_plots, dir_work, dir_gain, dir_rms, dir_status = dir_names(dirrepo, panel_id) fname_prefix, panel_alias = file_name_prefix(dirrepo, dettype, panel_id, _tstamp, exp, irun) prefix_offset, prefix_peds, prefix_plots, prefix_gain, prefix_rms, prefix_status =\ path_prefixes(fname_prefix, dir_offset, dir_peds, dir_plots, dir_gain, dir_rms, dir_status) #mpars = (('pedestals', 'pedestals', prefix_peds, dir_peds),\ # ('rms', 'pixel_rms', prefix_rms, dir_rms),\ # ('status', 'pixel_status', prefix_status, dir_status),\ # ('gain', 'pixel_gain', prefix_gain, dir_gain)) mpars = [] if 'p' in proc: mpars.append(('pedestals', 'pedestals', prefix_peds, dir_peds)) if 'r' in proc: mpars.append(('rms', 'pixel_rms', prefix_rms, dir_rms)) if 's' in proc: mpars.append(('status', 'pixel_status', prefix_status, dir_status)) if 'g' in proc: mpars.append(('gain', 'pixel_gain', prefix_gain, dir_gain)) if 'c' in proc: mpars.append(('gainci', 'pixel_gain', prefix_gain, dir_gain)) if 'c' in proc: add_links_for_gainci_fixed_modes(dir_gain, fname_prefix) # FH->AHL-H, FM->AML-M, FL->AML-L/AHL-L for (ctype, octype, prefix, dir_ctype) in mpars: fmt = CTYPE_FMT.get(octype,'%.5f') nda_dtype = CTYPE_DTYPE.get(octype, np.float32) logger.debug('begin merging for ctype:%s, octype:%s, fmt:%s,\n prefix:%s' % (ctype, octype, fmt, prefix)) fname = '%s_%s.txt' % (prefix, ctype) nda = merge_panel_gain_ranges(dir_ctype, panel_id, ctype, _tstamp, shape, nda_dtype, fname, fmt, filemode) if octype in dic_consts: dic_consts[octype].append(nda) # append for panel per ctype else: dic_consts[octype] = [nda,] logger.info('\n%s\nMERGE PANEL CONSTANTS AND DEPLOY THEM\n' % (80*'_')) #if deploy: dmerge = dir_merge(dirrepo) create_directory(dmerge, mode=dirmode) fmerge_prefix = fname_prefix_merge(dmerge, detname, _tstamp, exp, irun) for octype, lst in dic_consts.items(): mrg_nda = merge_panels(lst) logger.info(info_ndarr(mrg_nda, 'merged constants for %s ' % octype)) fmerge = '%s-%s.txt' % (fmerge_prefix, octype) fmt = CTYPE_FMT.get(octype,'%.5f') save_ndarray_in_textfile(mrg_nda, fmerge, filemode, fmt) if True: # deploy: # check opt "-t" if constants need to be deployed with diffiernt time stamp or run number use_external_run = tstamp is not None and tstamp<10000 use_external_ts = tstamp is not None and tstamp>9999 tvalid_sec = time_sec_from_stamp(fmt=cc.TSFORMAT_SHORT, time_stamp=str(tstamp))\ if use_external_ts else cpdic.get('trun_sec', None) ivalid_run = tstamp if use_external_run else irun\ if not use_external_ts else 0 dtype = 'ndarray' kwa = { 'iofname' : fmerge, 'experiment' : exp, 'ctype' : octype, 'dtype' : dtype, 'detector' : detname, 'detname' : det_name, 'longname' : longname, 'time_sec' : tvalid_sec, 'time_stamp' : str_tstamp(fmt=cc.TSFORMAT, time_sec=int(tvalid_sec)), 'tsshort' : str_tstamp(fmt=cc.TSFORMAT_SHORT, time_sec=int(tvalid_sec)), 'tstamp_orig': cpdic.get('tsrun_dark', None), 'run' : ivalid_run, 'run_end' : run_end, 'run_orig' : irun, 'version' : version, 'comment' : comment, 'extpars' : {'content':'extended parameters dict->json->str',}, 'dettype' : dettype, 'dbsuffix' : dbsuffix } logger.debug('DEPLOY metadata: %s' % str(kwa)) data = mu.data_from_file(fmerge, octype, dtype, True) logger.debug(info_ndarr(data, 'merged constants loaded from file')) if deploy: resp = wu.deploy_constants(data, exp, longname, url=cc.URL_KRB, krbheaders=cc.KRBHEADERS, **kwa) #id_data_exp, id_data_det, id_doc_exp, id_doc_det = resp if resp is not None else: logger.warning('TO DEPLOY CONSTANTS ADD OPTION -D')