def log_file_name(lfpath) : """Returns (str) log file name like /reg/g/psdm/logs/calibman/lcls2/2018/20180518T122407-dubrovin.txt """ t0_sec = gu.time() tstamp = gu.str_tstamp('%Y%m%dT%H%M%S', t0_sec) #year_month = gu.str_tstamp('%Y/%m', time_sec=None) year = gu.str_tstamp('%Y', time_sec=None) return '%s/%s/%s-%s.txt' % (lfpath, year, tstamp, gu.get_login())#, os.getpid())
def print_experiments_count_runs(): # ins='CXI' d_ins_nruns = {} d_ins_nexps = {} nruns_tot = 0 nexps = 0 for ins in INSTRUMENTS: nruns_ins = 0 exps = experiments(ins) nexps += len(exps) for exp in exps: runs = runs_in_xtc_dir(exp) nruns = len(runs) nruns_ins += nruns nruns_tot += nruns print(' %10s nruns:%4d' % (exp, nruns)) d_ins_nruns[ins] = nruns_ins d_ins_nexps[ins] = len(exps) print('\nSummary on %s\n%s' % (gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time()), 40 * '_')) for ins, nruns in d_ins_nruns.items(): print('%6d runs in %4d experiments of %s' % (nruns, d_ins_nexps[ins], ins)) dname = '%s/<all-ins>/<all-exp>/' % DIR_INS print('%s\n%6d runs in %4d experiments of %s' % (40 * '_', nruns_tot, nexps, dname))
def save_txt(fname='nda.txt', arr=None, cmts=(), fmt='%.1f', verbos=False, addmetad=True) : """Save n-dimensional numpy array to text file with metadata. - fname - file name for text file, - arr - numpy array, - cmts -list of comments which will be saved in the file header. """ #recs = ['# %03d %s' % (i,cmt) for i, cmt in enumerate(cmts)] recs = ['# %s' % cmt for cmt in cmts] recs.append('\n# HOST %s' % gu.get_hostname()) recs.append('# WORK_DIR %s' % gu.get_cwd()) recs.append('# FILE_NAME %s' % fname) recs.append('# DATE_TIME %s' % gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S')) recs.append('# UID %s' % gu.get_login()) recs.append('# SHAPE %s' % str(arr.shape).replace(' ','')) recs.append('# DATATYPE %s' % str(arr.dtype)) if addmetad : recs.append('\n# DTYPE %s' % str(arr.dtype)) recs.append('# NDIM %s' % len(arr.shape)) for i in range(len(arr.shape)) : recs.append('# DIM:%d %s' % (i, arr.shape[i])) arr2d = nu.reshape_nda_to_2d(arr) # pretty formatting recs.append('' if len(arr.shape)>1 else '\n') nline = '\n' if len(arr.shape)>1 else ' ' hdr = '\n'.join(recs) #print(hdr) np.savetxt(fname, arr, fmt, delimiter=' ', newline=nline, header=hdr, comments='') #, footer='\n') #, comments='# ') if verbos : print('File %s is saved' % fname)
def exportdb(self): """Exports database. Equivalent to: mongodump -d <dbname> -o <filename> mongodump --host psanaphi105 --port 27017 --db calib-cxi12345 --archive=db.20180122.arc """ host, port, dbname, fname = self.host_port_dbname_fname() tstamp = gu.str_tstamp(fmt='%Y-%m-%dT%H-%M-%S') fname = 'cdb-%s-%s.arc' % (tstamp, dbname) if fname is None else fname dbu.exportdb(host, port, dbname, fname)
def deploy_calib_array(cdir, src, type, run_start, run_end=None, arr=None, dcmts={}, fmt='%.1f', pbits=1): """Deploys array in calibration file - makes the new file name using make_calib_file_name(...) - if file with this name already exists - rename it with current timestamp in the name - save array in file - add history record """ fname = make_calib_file_name(cdir, src, type, run_start, run_end, pbits) path_history = '%s/HISTORY' % os.path.dirname(fname) if os.path.exists(fname): fname_bkp = '%s-%s' % (fname, gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S')) os.system('cp %s %s' % (fname, fname_bkp)) if pbits & 1: print('Existing file %s\nis backed-up %s' % (fname, fname_bkp)) # extend dictionary for other parameters d = dict(dcmts) d['run'] = run_start d['fname'] = os.path.basename(fname) d['src'] = src d['ctype'] = type # make list of comments cmts = ['%s %s' % (k.upper().ljust(11), v) for k, v in d.iteritems()] # save n-dimensional numpy array in the tmp text file fntmp = tempfile.NamedTemporaryFile(mode='r+b', suffix='.data') if pbits & 2: print('Save constants in tmp file: %s' % fntmp.name) save_txt(fntmp.name, arr, cmts, fmt='%.1f') if pbits & 1: print('Deploy constants in file: %s' % fname) # USE cat in stead of cp and move in order to create output file with correct ACL permissions cmd_cat = 'cat %s > %s' % (fntmp.name, fname) #os.system(cmd_cat) stream = os.popen(cmd_cat) resp = stream.read() msg = 'Command: %s\n - resp: %s' % (cmd_cat, resp) if pbits & 2: print(msg) # add record to the HISTORY file hrec = _history_record(d) if pbits & 1: print('Add record: %sto the file: %s' % (hrec, path_history)) gu.save_textfile(hrec, path_history, mode='a')
def msg_to_log(runs=[]): """Returns (str) message to the log file for list of (str) runs. """ if len(runs) == 0: return None tstamp = gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time()) login = gu.get_login() cwd = gu.get_cwd() host = gu.get_hostname() cmd = sys.argv[0].split('/')[-1] recs = [ '%s %s %s %s cwd:%s cmd:%s' % (s, tstamp, login, host, cwd, cmd) for s in runs ] text = '\n'.join(recs) return text + '\n'
def test_insert_constants(expname=TEST_EXPNAME, detname=TEST_DETNAME, ctype='test_ctype', runnum=10, data='test text sampele'): """ Inserts constants using direct MongoDB interface from MDBUtils. """ import psana.pyalgos.generic.Utils as gu print('test_delete_database 1:', database_names()) #txt = '%s\nThis is a string\n to test\ncalibration storage' % gu.str_tstamp() #data, ctype = txt, 'testtext'; logger.debug('txt: %s' % str(data)) #data, ctype = get_test_nda(), 'testnda'; logger.debug(info_ndarr(data, 'nda')) #data, ctype = get_test_dic(), 'testdict'; logger.debug('dict: %s' % str(data)) kwa = {'user' : gu.get_login()} t0_sec = time() ts = gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S%z', time_sec=t0_sec) mu.insert_constants('%s - saved at %s'%(data,ts), expname, detname, ctype, runnum+int(tname), int(t0_sec),\ time_stamp=ts, **kwa) print('test_delete_database 2:', database_names())
def deploy_calib_file(cdir, src, type, run_start, run_end=None, ifname='', dcmts={}, pbits=1): """Deploys calibration file - makes the new file name using make_calib_file_name(...) - if file with this name already exists - rename it with current timestamp in the name - save array in file - add history record """ fname = make_calib_file_name(cdir, src, type, run_start, run_end, pbits) path_history = '%s/HISTORY' % os.path.dirname(fname) if os.path.exists(fname): fname_bkp = '%s-%s' % (fname, gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S')) os.system('cp %s %s' % (fname, fname_bkp)) if pbits & 1: print('Existing file %s\nis backed-up %s' % (fname, fname_bkp)) # extend dictionary for other parameters d = dict(dcmts) d['run'] = run_start d['fname'] = os.path.basename(fname) d['ifname'] = ifname d['src'] = src d['ctype'] = type if pbits & 1: print('Deploy constants in file: %s' % fname) # USE cat in stead of cp and move in order to create output file with correct ACL permissions cmd_cat = 'cat %s > %s' % (ifname, fname) #os.system(cmd_cat) stream = os.popen(cmd_cat) resp = stream.read() msg = 'Command: %s\n - resp: %s' % (cmd_cat, resp) if pbits & 2: print(msg) # add record to the HISTORY file hrec = _history_record(d) if pbits & 1: print('Add record: %sto the file: %s' % (hrec, path_history)) gu.save_textfile(hrec, path_history, mode='a')
def do_work(): prefix = './%s-figs-ti_vs_tj' % gu.str_tstamp( fmt='%Y-%m-%d', time_sec=None) # '%Y-%m-%dT%H:%M:%S%z' gu.create_directory(prefix, mode=0o775) path = os.path.abspath(os.path.dirname(__file__)) print('path to npy flies dir:', path) ti_vs_tj = np.load('%s/ti_vs_tj.npy' % path) t_all = np.load('%s/t_all.npy' % path) trange = (1400., 2900.) print_ndarr(ti_vs_tj, 'ti_vs_tj:\n') print_ndarr(t_all, 't_all:\n') sum_bkg = t_all.sum() sum_cor = ti_vs_tj.sum() print('sum_bkg:', sum_bkg) print('sum_cor:', sum_cor) imrange = trange + trange # (1400., 2900., 1400., 2900.) axim = gr.plotImageLarge(ti_vs_tj, img_range=imrange, amp_range=(0,500), figsize=(11,10),\ title='ti_vs_tj', origin='lower', window=(0.10, 0.08, 0.88, 0.88), cmap='inferno') gr.save('%s/fig-ti_vs_tj.png' % prefix) bkg = np.outer(t_all, t_all) / sum_bkg print_ndarr(bkg, 'bkg:\n') axim = gr.plotImageLarge(bkg, img_range=imrange, amp_range=(0,500), figsize=(11,10),\ title='bkg', origin='lower', window=(0.10, 0.08, 0.88, 0.88), cmap='inferno') gr.save('%s/fig-ti_vs_tj-bkg.png' % prefix) harr = t_all nbins = harr.size ht = HBins(trange, nbins, vtype=np.float32) # ht.binedges() fig, axhi, hi = gr.hist1d(ht.bincenters(), bins=nbins, amp_range=ht.limits(), weights=harr, color='b', show_stat=True,\ log=True, figsize=(7,6), axwin=(0.10, 0.10, 0.88, 0.85), title='1-d bkg',\ xlabel='time of all hits (ns)', ylabel='number of hits', titwin='1-d bkg') gr.save('%s/fig-time-hits.png' % prefix) gr.show()
def _history_record(dcmts): """Returns history record made of dictionary comments and system info """ user = gu.get_login() host = gu.get_hostname() tstamp = gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S zone:%Z') rnum = '%04d' % dcmts.get('run') exp = '%s' % dcmts.get('exp') ifname = '%s' % dcmts.get('ifname') ofname = '%s' % dcmts.get('fname') app = '%s' % dcmts.get('app') cmt = '%s' % dcmts.get('comment') return 'file:%s copy_of:%s exp:%s run:%s app:%s user:%s host:%s cptime:%s comment:%s\n' % \ (ofname.ljust(14), ifname, exp.ljust(8), rnum.ljust(4), app.ljust(10), user, host, tstamp.ljust(29), cmt)
def time_and_timestamp(**kwargs): """Returns "time_sec" and "time_stamp" from **kwargs. If one of these parameters is missing, another is reconstructed from available one. If both missing - current time is used. """ time_sec = kwargs.get('time_sec', None) time_stamp = kwargs.get('time_stamp', None) if time_sec is not None: assert isinstance( time_sec, str), 'time_and_timestamp - parameter time_sec should be str' int_time_sec = int(time_sec) assert 0 < int_time_sec < 5000000000, 'time_and_timestamp - parameter time_sec should be in allowed range' if time_stamp is None: time_stamp = gu.str_tstamp(TSFORMAT, int_time_sec) else: if time_stamp is None: time_sec, time_stamp = gu.time_and_stamp(TSFORMAT) else: time_sec = gu.time_sec_from_stamp(TSFORMAT, time_stamp) return str(time_sec), time_stamp
def _timestamp(time_sec) -> str: """Converts time_sec in timestamp of adopted format TSFORMAT. """ return gu.str_tstamp(TSFORMAT, int(time_sec))
def get_test_txt(): """Returns text for test purpose. """ return '%s\nThis is a string\n to test\ncalibration storage' % gu.str_tstamp( )
def proc_control(parser): """Dataset processing module - Submits job in batch for data processing - Checks in loop batch job status - Marks in DB that job is processed - Save common log file for submission and processing """ (popts, pargs) = parser.parse_args() exp = popts.exp run = popts.run procname = popts.pro qname = popts.que dt_sec = popts.dts sources = popts.srs tstamp = gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time()) logpref = rpu.log_batch_prefix(exp, run, procname) logfname = '%s-%s.txt' % (logpref, tstamp) for i in range(10): if gu.create_path(logfname, depth=6, mode=0774, verb=False): continue gu.save_textfile('\nCreated path: %s' % logfname, logfname, mode='a') os.chmod(logfname, 0664) ofprefix = '%s/nda-#exp-#run-#src-#evts-#type.txt' % rpu.work_dir( exp, procname) gu.save_textfile('\nOutput work files: %s' % ofprefix, logfname, mode='a') for i in range(5): if gu.create_path(ofprefix, depth=8, mode=0774, verb=False): continue msg = '\nproc_control exp: %s run: %s procname: %s qname: %s logfname %s'%\ (exp, str(run), procname, qname, logfname) gu.save_textfile(str(msg), logfname, mode='a') cmd = _batch_submit_command(exp, run, procname, qname, logfname, ofprefix, sources) if cmd is None: raise IOError('ERROR: batch submission command is None...') t0_sec = time() out, err, jobid = spu.batch_job_submit(cmd, env=None, shell=False) msg = 'bsub subproc time dt=%7.3f sec' % (time() - t0_sec) gu.save_textfile(msg, logfname, mode='a') #if 'submitted without an AFS token' in err : # print ' Tip: to get rid of error message use commands: kinit; aklog' rec = gu.log_rec_on_start() msg = '%s\nSubmitted batch job %s to %s\n cmd: %s\n out: %s\n err: "%s"\n%s\n'%\ (rec, jobid, qname, cmd, out, err.strip('\n'), 80*'_') #print msg gu.save_textfile(msg, logfname, mode='a') # check batch job status in loop status = None counter = 0 while status in (None, 'RUN', 'PEND'): counter += 1 out, err, status = spu.batch_job_status(jobid, user=None, qname=qname) ts = gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time()) msg = '%4d %s %s job %s status %s' % (counter, ts, qname, jobid, status) #print msg gu.save_textfile('%s\n' % msg, logfname, mode='a') if status in ('EXIT', 'DONE'): break sleep(dt_sec) # change log file name in case of bad status if status != 'DONE': logfname_bad = '%s-%s' % (logfname, str(status)) #cmd = 'mv %s %s' % (logfname, logfname_bad) #os.system(cmd) os.rename(logfname, logfname_bad)