def deploy_calib_array(cdir, src, type, run_start, run_end=None, arr=None, dcmts={}, fmt='%.1f', pbits=1): """Deploys array in calibration file - makes the new file name using make_calib_file_name(...) - if file with this name already exists - rename it with current timestamp in the name - save array in file - add history record """ fname = make_calib_file_name(cdir, src, type, run_start, run_end, pbits) path_history = '%s/HISTORY' % os.path.dirname(fname) if os.path.exists(fname): fname_bkp = '%s-%s' % (fname, gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S')) os.system('cp %s %s' % (fname, fname_bkp)) if pbits & 1: print('Existing file %s\nis backed-up %s' % (fname, fname_bkp)) # extend dictionary for other parameters d = dict(dcmts) d['run'] = run_start d['fname'] = os.path.basename(fname) d['src'] = src d['ctype'] = type # make list of comments cmts = ['%s %s' % (k.upper().ljust(11), v) for k, v in d.iteritems()] # save n-dimensional numpy array in the tmp text file fntmp = tempfile.NamedTemporaryFile(mode='r+b', suffix='.data') if pbits & 2: print('Save constants in tmp file: %s' % fntmp.name) save_txt(fntmp.name, arr, cmts, fmt='%.1f') if pbits & 1: print('Deploy constants in file: %s' % fname) # USE cat in stead of cp and move in order to create output file with correct ACL permissions cmd_cat = 'cat %s > %s' % (fntmp.name, fname) #os.system(cmd_cat) stream = os.popen(cmd_cat) resp = stream.read() msg = 'Command: %s\n - resp: %s' % (cmd_cat, resp) if pbits & 2: print(msg) # add record to the HISTORY file hrec = _history_record(d) if pbits & 1: print('Add record: %sto the file: %s' % (hrec, path_history)) gu.save_textfile(hrec, path_history, mode='a')
def save_constants_in_file(tname, fname='0-end.txt'): if tname == '0': nda, ctype = delayed_denom() print_ndarr(nda, 'nda for ctype: %s' % ctype) save_txt(fname, nda, cmts=(), fmt='%d', verbos=True, addmetad=True) cmd = 'cdb add -e tmo -d tmott -c %s -r 1 -f %s -u dubrovin' % (ctype, fname) elif tname == '1': d, ctype = fir_coefficients() print('dict:', d) save_textfile(str(d), fname, mode='w', verb=True) cmd = 'cdb add -e tmo -d tmott -c %s -r 1 -f %s -i txt -u dubrovin' % ( ctype, fname) msg = 'DO NOT FORGET ADD CONSTS TO DB: %s' % cmd print(msg)
def save_data_in_file(data, prefix, control={ 'txt': True, 'npy': True }, fmt='%.3f'): #elif data_type == 'any': # gu.save_textfile(str(data), fname, mode='w', verb=verb) if isinstance(data, np.ndarray): if control['txt']: from psana.pscalib.calib.NDArrIO import save_txt # load_txt fname = '%s.txt' % prefix save_txt(fname, data, cmts=(), fmt=fmt) if control['npy']: fname = '%s.npy' % prefix np.save(fname, data, allow_pickle=False) logger.info('saved file: %s' % fname) else: logger.warning('DO NOT SAVE unexpected data type: %s' % type(data))
def save_doc(self): """Saves document metadata and data in files """ logger.debug('In save_doc') wdoce = cp.cmwdbdoceditor if wdoce is None or wdoce.data_nda is None: logger.warning( 'Document editor is not available. Select collection in DB then document in List mode.' ) return dicdoc = wdoce.get_model_dicdoc(discard_id_ts=False) nda = wdoce.get_data_nda() msg = '\n '.join( ['%12s : %s' % (k, dicdoc[k]) for k in sorted(dicdoc.keys())]) logger.info('Save array and document metadata in files\n%s \n%s' % (msg, info_ndarr(nda, 'data n-d array '))) prefix = self.out_fname_prefix(dicdoc) d = {'data': True, 'meta': True} resp = change_check_box_dict_in_popup_menu(d, 'Select and confirm',\ msg='Save current document in file\n%s\nfor types:'%prefix, parent=self.but_add) if resp == 1: logger.info( 'Save document data and metadata in files with prefix: %s' % prefix) for k in d.keys(): if d[k]: fname = '%s.%s' % (prefix, k) logger.debug('save_doc %s in file %s' % (k, fname)) if k == 'data': save_txt(fname, nda, cmts=(), fmt='%.1f') if k == 'meta': save_textfile(msg, fname, mode='w') else: logger.warning('Command "Save" is cancelled')
def get(self): """Saves in file calibration constants from database. """ mode, kwargs = self.mode, self.kwargs defs = self.defs host = kwargs.get('host', None) port = kwargs.get('port', None) exp = kwargs.get('experiment', None) det = kwargs.get('detector', None) ctype = kwargs.get('ctype', None) run = kwargs.get('run', None) run_end = kwargs.get('run_end', None) tsec = kwargs.get('time_sec', None) tstamp = kwargs.get('time_stamp', None) vers = kwargs.get('version', None) fname = kwargs.get('iofname', None) query = {'detector': det, 'ctype': ctype} if run != defs['run']: query['run'] = {'$lte': run} query['run_end'] = {'$gte': run} if tsec != defs['time_sec']: query['time_sec'] = {'$lte': tsec} if vers != defs['version']: query['version'] = vers #logger.debug('query: %s' % str(query)) dbname = dbu.get_dbname(**kwargs) client = self.client() if not self.check_database(client, dbname): return detname = kwargs.get('detector', None) if detname is None: logger.warning( '%s needs in the collection name. Please specify the detector name.' % (mode)) colname = detname db, fs = dbu.db_and_fs(client, dbname) colnames = dbu.collection_names(db) if not (colname in colnames): # dbu.collection_exists(db, colname) logger.warning('db "%s" does not have collection "%s"' % (db.name, str(colname))) return col = dbu.collection(db, colname) logger.info('Search document in db "%s" collection "%s"' % (dbname, colname)) #client, expname, detname, db_exp, db_det, fs_exp, fs_det, col_exp, col_det =\ # dbu.connect(host=host, port=port, experiment=exp, detector=det, verbose=verb) #fs, doc = fs_exp, dbu.find_doc(col, query) #if doc is None : # fs, doc = fs_det, dbu.find_doc(col_det, query) # if doc is None : # logger.warning('Can not find document for query: %s' % str(query)) # return doc = dbu.find_doc(col, query) if doc is None: logger.warning('Can not find document for query: %s' % str(query)) return data = dbu.get_data_for_doc(fs, doc) if data is None: logger.warning('Can not load data for doc: %s' % str(doc)) return if fname is None: fname = 'clb-%s-%s-%s.npy' % (expname, det, ctype) if ctype == 'geometry': gu.save_textfile(data, fname, mode='w', verb=verb) elif os.path.splitext(fname)[1] == '.npy': np.save(fname, data, allow_pickle=False) else: save_txt(fname, data, fmt='%.2f') if verb: logger.info(info_ndarr(data, 'nda', first=0, last=3)) logger.info('Save constants in file: %s' % fname)
def summary(self, evt): logger.info('%s\nRaw data for %s found/selected in %d events' % (80 * '_', self.detname, self.counter)) if self.counter: logger.info(', begin data summary stage') else: logger.info(', STOP processing, there are no arrays to save...') return t0_sec = time() # make shorter references det = self.det ofname = self.ofname plotim = self.plotim savebw = self.savebw int_hi = self.int_hi int_lo = self.int_lo fraclm = self.fraclm counter = self.counter arr_av1 = divide_protected(self.arr_sum1, self.arr_sum0) arr_av2 = divide_protected(self.arr_sum2, self.arr_sum0) frac_int_lo = np.array(self.sta_int_lo / counter, dtype=np.float32) frac_int_hi = np.array(self.sta_int_hi / counter, dtype=np.float32) arr_rms = np.sqrt(arr_av2 - np.square(arr_av1)) rms_min, rms_max = evaluate_limits(arr_rms, self.rmsnlo, self.rmsnhi, self.rms_lo, self.rms_hi, cmt='RMS') ave_min, ave_max = evaluate_limits(arr_av1, self.intnlo, self.intnhi, self.int_lo, self.int_hi, cmt='AVE') arr_sta_rms_hi = np.select((arr_rms > rms_max, ), (self.arr1, ), 0) arr_sta_rms_lo = np.select((arr_rms < rms_min, ), (self.arr1, ), 0) arr_sta_int_hi = np.select((frac_int_hi > fraclm, ), (self.arr1, ), 0) arr_sta_int_lo = np.select((frac_int_lo > fraclm, ), (self.arr1, ), 0) arr_sta_ave_hi = np.select((arr_av1 > ave_max, ), (self.arr1, ), 0) arr_sta_ave_lo = np.select((arr_av1 < ave_min, ), (self.arr1, ), 0) logger.info(' Bad pixel status:') logger.info(' status 1: %8d pixel rms > %.3f' % (arr_sta_rms_hi.sum(), rms_max)) logger.info(' status 8: %8d pixel rms < %.3f' % (arr_sta_rms_lo.sum(), rms_min)) logger.info( ' status 2: %8d pixel intensity > %g in more than %g fraction of events' % (arr_sta_int_hi.sum(), int_hi, fraclm)) logger.info( ' status 4: %8d pixel intensity < %g in more than %g fraction of events' % (arr_sta_int_lo.sum(), int_lo, fraclm)) logger.info(' status 16: %8d pixel average > %g' % (arr_sta_ave_hi.sum(), ave_max)) logger.info(' status 32: %8d pixel average < %g' % (arr_sta_ave_lo.sum(), ave_min)) #0/1/2/4/8/16/32 for good/hot-rms/saturated/cold/cold-rms/average above limit/average below limit, arr_sta = np.zeros(self.shape, dtype=np.int64) arr_sta += arr_sta_rms_hi # hot rms arr_sta += arr_sta_rms_lo * 8 # cold rms arr_sta += arr_sta_int_hi * 2 # satturated arr_sta += arr_sta_int_lo * 4 # cold arr_sta += arr_sta_ave_hi * 16 # too large average arr_sta += arr_sta_ave_lo * 32 # too small average arr_msk = np.select((arr_sta > 0, ), (self.arr0, ), 1) cmod = None # self._common_mode_pars(arr_av1, arr_rms, arr_msk) if plotim & 1: plot_det_image(det, evt, arr_av1, tit='average') if plotim & 2: plot_det_image(det, evt, arr_rms, tit='RMS') if plotim & 4: plot_det_image(det, evt, arr_sta, tit='status') if plotim & 8: plot_det_image(det, evt, arr_msk, tit='mask') if plotim & 16: plot_det_image(det, evt, self.arr_max, tit='maximum') if plotim & 32: plot_det_image(det, evt, self.arr_min, tit='minimum') if plotim & 64: plot_det_image(det, evt, self.sta_int_lo, tit='statistics below threshold') if plotim & 128: plot_det_image(det, evt, self.sta_int_hi, tit='statistics above threshold') cmts = ['DATASET %s' % self.detname, 'STATISTICS %d' % counter] # Save n-d array in text file % template = fname_template(self.orun, self.detname, ofname, counter, self.ev1_sec, self.ev1_nsec) addmetad = True verbos = self.verbos if savebw & 1: save_txt(template % 'ave', arr_av1, cmts + ['ARR_TYPE average'], '%8.2f', verbos, addmetad) if savebw & 2: save_txt(template % 'rms', arr_rms, cmts + ['ARR_TYPE RMS'], '%8.2f', verbos, addmetad) if savebw & 4: save_txt(template % 'sta', arr_sta, cmts + ['ARR_TYPE status'], '%d', verbos, addmetad) if savebw & 8: save_txt(template % 'msk', arr_msk, cmts + ['ARR_TYPE mask'], '%1d', verbos, addmetad) if savebw & 16: save_txt(template % 'max', self.arr_max, cmts + ['ARR_TYPE max'], '%d', verbos, addmetad) if savebw & 32: save_txt(template % 'min', self.arr_min, cmts + ['ARR_TYPE min'], '%d', verbos, addmetad) if savebw & 64 and cmod is not None: np.savetxt(template % 'cmo', cmod, fmt='%d', delimiter=' ', newline=' ') save_txt(template % 'cmm', cmod, cmts + ['ARR_TYPE common_mode'], '%d', verbos, False) args = vars(self.args) version = args.get('version', str_tstamp('V%Y%m%dT%H%M%S', t0_sec)) orun = self.orun loglev = args.get('loglev', 'INFO') # args.expnam # args.runnum, orun.timestamp kwa = { 'detname': self.detname, 'experiment': orun.expt, 'runnum': orun.runnum, 'timestamp': str_tstamp(time_sec=self.ev1_sec), 'version': version, 'comment': str(cmts), 'loglev': loglev, } logger.debug('kwa:', kwa) logger.debug('args:', self.args) if self.addcdb: from psana.pscalib.calib.MDBCommands import add_constants if savebw & 1: add_constants(template % 'ave', 'pedestals', **kwa) if savebw & 2: add_constants(template % 'rms', 'pixel_rms', **kwa) if savebw & 4: add_constants(template % 'sta', 'pixel_status', **kwa) if savebw & 8: add_constants(template % 'msk', 'pixel_mask', **kwa) if savebw & 16: add_constants(template % 'max', 'pixel_max', **kwa) if savebw & 32: add_constants(template % 'min', 'pixel_min', **kwa) logger.info('Data summary for %s is completed, dt=%7.3f sec' % (self.detname, time() - t0_sec))
def save_ndarray_in_textfile(nda, fname, fmode, fmt): fexists = os.path.exists(fname) save_txt(fname=fname, arr=nda, fmt=fmt) if not fexists: set_file_access_mode(fname, fmode) logger.debug('saved: %s fmode: %s fmt: %s' % (fname, oct(fmode), fmt))
def summary(self, evt): logger.info('%s\nRaw data for %s found/selected in %d events' % (80 * '_', self.detname, self.counter)) if self.counter: logger.info(', begin data summary stage') else: logger.info(', STOP processing, there are no arrays to save...') return t0_sec = time() # make shorter references det = self.det ofname = self.ofname plotim = self.plotim savebw = self.savebw verbos = self.verbos int_hi = self.int_hi int_lo = self.int_lo fraclm = self.fraclm counter = self.counter arr_av1 = divide_protected(self.arr_sum1, self.arr_sum0) arr_av2 = divide_protected(self.arr_sum2, self.arr_sum0) frac_int_lo = np.array(self.sta_int_lo / counter, dtype=np.float32) frac_int_hi = np.array(self.sta_int_hi / counter, dtype=np.float32) arr_rms = np.sqrt(arr_av2 - np.square(arr_av1)) rms_min, rms_max = evaluate_limits(arr_rms, self.rmsnlo, self.rmsnhi, self.rms_lo, self.rms_hi, self.verbos, cmt='RMS') #if self.rms_lo is None or self.rms_hi is None or self.rms_hi == 0.\ #else (self.rms_lo, self.rms_hi) #if rms_min<0 : rms_min=0 ave_min, ave_max = evaluate_limits(arr_av1, self.intnlo, self.intnhi, self.int_lo, self.int_hi, self.verbos, cmt='AVE')\ arr_sta_rms_hi = np.select((arr_rms > rms_max, ), (self.arr1, ), 0) arr_sta_rms_lo = np.select((arr_rms < rms_min, ), (self.arr1, ), 0) arr_sta_int_hi = np.select((frac_int_hi > fraclm, ), (self.arr1, ), 0) arr_sta_int_lo = np.select((frac_int_lo > fraclm, ), (self.arr1, ), 0) arr_sta_ave_hi = np.select((arr_av1 > ave_max, ), (self.arr1, ), 0) arr_sta_ave_lo = np.select((arr_av1 < ave_min, ), (self.arr1, ), 0) logger.info(' Bad pixel status:') logger.info(' status 1: %8d pixel rms > %.3f' % (arr_sta_rms_hi.sum(), rms_max)) logger.info(' status 8: %8d pixel rms < %.3f' % (arr_sta_rms_lo.sum(), rms_min)) logger.info( ' status 2: %8d pixel intensity > %g in more than %g fraction of events' % (arr_sta_int_hi.sum(), int_hi, fraclm)) logger.info( ' status 4: %8d pixel intensity < %g in more than %g fraction of events' % (arr_sta_int_lo.sum(), int_lo, fraclm)) logger.info(' status 16: %8d pixel average > %g' % (arr_sta_ave_hi.sum(), ave_max)) logger.info(' status 32: %8d pixel average < %g' % (arr_sta_ave_lo.sum(), ave_min)) #0/1/2/4/8/16/32 for good/hot-rms/saturated/cold/cold-rms/average above limit/average below limit, arr_sta = np.zeros(self.shape, dtype=np.int64) arr_sta += arr_sta_rms_hi # hot rms arr_sta += arr_sta_rms_lo * 8 # cold rms arr_sta += arr_sta_int_hi * 2 # satturated arr_sta += arr_sta_int_lo * 4 # cold arr_sta += arr_sta_ave_hi * 16 # too large average arr_sta += arr_sta_ave_lo * 32 # too small average arr_msk = np.select((arr_sta > 0, ), (self.arr0, ), 1) cmod = None # self._common_mode_pars(arr_av1, arr_rms, arr_msk) if plotim & 1: plot_det_image(det, evt, arr_av1, tit='average') if plotim & 2: plot_det_image(det, evt, arr_rms, tit='RMS') if plotim & 4: plot_det_image(det, evt, arr_sta, tit='status') if plotim & 8: plot_det_image(det, evt, arr_msk, tit='mask') if plotim & 16: plot_det_image(det, evt, self.arr_max, tit='maximum') if plotim & 32: plot_det_image(det, evt, self.arr_min, tit='minimum') if plotim & 64: plot_det_image(det, evt, self.sta_int_lo, tit='statistics below threshold') if plotim & 128: plot_det_image(det, evt, self.sta_int_hi, tit='statistics above threshold') cmts = ['DATASET %s' % self.detname, 'STATISTICS %d' % counter] # Save n-d array in text file % template = fname_template(self.orun, self.detname, ofname, counter, self.ev1_sec, self.ev1_nsec) addmetad = True if savebw & 1: save_txt(template % 'ave', arr_av1, cmts + ['ARR_TYPE average'], '%8.2f', verbos, addmetad) if savebw & 2: save_txt(template % 'rms', arr_rms, cmts + ['ARR_TYPE RMS'], '%8.2f', verbos, addmetad) if savebw & 4: save_txt(template % 'sta', arr_sta, cmts + ['ARR_TYPE status'], '%d', verbos, addmetad) if savebw & 8: save_txt(template % 'msk', arr_msk, cmts + ['ARR_TYPE mask'], '%1d', verbos, addmetad) if savebw & 16: save_txt(template % 'max', self.arr_max, cmts + ['ARR_TYPE max'], '%d', verbos, addmetad) if savebw & 32: save_txt(template % 'min', self.arr_min, cmts + ['ARR_TYPE min'], '%d', verbos, addmetad) if savebw & 64 and cmod is not None: np.savetxt(template % 'cmo', cmod, fmt='%d', delimiter=' ', newline=' ') save_txt(template % 'cmm', cmod, cmts + ['ARR_TYPE common_mode'], '%d', verbos, False) if self.verbos & 1: logger.info('Data summary for %s is completed, dt=%7.3f sec' % (self.detname, time() - t0_sec))