def log_file_name(lfpath) :
"""Returns (str) log file name like /reg/g/psdm/logs/calibman/lcls2/2018/20180518T122407-dubrovin.txt
"""
t0_sec = gu.time()
tstamp = gu.str_tstamp('%Y%m%dT%H%M%S', t0_sec)
#year_month = gu.str_tstamp('%Y/%m', time_sec=None)
year = gu.str_tstamp('%Y', time_sec=None)
return '%s/%s/%s-%s.txt' % (lfpath, year, tstamp, gu.get_login())#, os.getpid())
def print_experiments_count_runs(): # ins='CXI'
d_ins_nruns = {}
d_ins_nexps = {}
nruns_tot = 0
nexps = 0
for ins in INSTRUMENTS:
nruns_ins = 0
exps = experiments(ins)
nexps += len(exps)
for exp in exps:
runs = runs_in_xtc_dir(exp)
nruns = len(runs)
nruns_ins += nruns
nruns_tot += nruns
print(' %10s nruns:%4d' % (exp, nruns))
d_ins_nruns[ins] = nruns_ins
d_ins_nexps[ins] = len(exps)
print('\nSummary on %s\n%s' %
(gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time()), 40 * '_'))
for ins, nruns in d_ins_nruns.items():
print('%6d runs in %4d experiments of %s' %
(nruns, d_ins_nexps[ins], ins))
dname = '%s/<all-ins>/<all-exp>/' % DIR_INS
print('%s\n%6d runs in %4d experiments of %s' %
(40 * '_', nruns_tot, nexps, dname))
def save_txt(fname='nda.txt', arr=None, cmts=(), fmt='%.1f', verbos=False, addmetad=True) :
"""Save n-dimensional numpy array to text file with metadata.
- fname - file name for text file,
- arr - numpy array,
- cmts -list of comments which will be saved in the file header.
"""
#recs = ['# %03d %s' % (i,cmt) for i, cmt in enumerate(cmts)]
recs = ['# %s' % cmt for cmt in cmts]
recs.append('\n# HOST %s' % gu.get_hostname())
recs.append('# WORK_DIR %s' % gu.get_cwd())
recs.append('# FILE_NAME %s' % fname)
recs.append('# DATE_TIME %s' % gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S'))
recs.append('# UID %s' % gu.get_login())
recs.append('# SHAPE %s' % str(arr.shape).replace(' ',''))
recs.append('# DATATYPE %s' % str(arr.dtype))
if addmetad :
recs.append('\n# DTYPE %s' % str(arr.dtype))
recs.append('# NDIM %s' % len(arr.shape))
for i in range(len(arr.shape)) :
recs.append('# DIM:%d %s' % (i, arr.shape[i]))
arr2d = nu.reshape_nda_to_2d(arr)
# pretty formatting
recs.append('' if len(arr.shape)>1 else '\n')
nline = '\n' if len(arr.shape)>1 else ' '
hdr = '\n'.join(recs)
#print(hdr)
np.savetxt(fname, arr, fmt, delimiter=' ', newline=nline, header=hdr, comments='') #, footer='\n') #, comments='# ')
if verbos : print('File %s is saved' % fname)
def exportdb(self):
"""Exports database. Equivalent to: mongodump -d <dbname> -o <filename>
mongodump --host psanaphi105 --port 27017 --db calib-cxi12345 --archive=db.20180122.arc
"""
host, port, dbname, fname = self.host_port_dbname_fname()
tstamp = gu.str_tstamp(fmt='%Y-%m-%dT%H-%M-%S')
fname = 'cdb-%s-%s.arc' % (tstamp, dbname) if fname is None else fname
dbu.exportdb(host, port, dbname, fname)
def deploy_calib_array(cdir,
src,
type,
run_start,
run_end=None,
arr=None,
dcmts={},
fmt='%.1f',
pbits=1):
"""Deploys array in calibration file
- makes the new file name using make_calib_file_name(...)
- if file with this name already exists - rename it with current timestamp in the name
- save array in file
- add history record
"""
fname = make_calib_file_name(cdir, src, type, run_start, run_end, pbits)
path_history = '%s/HISTORY' % os.path.dirname(fname)
if os.path.exists(fname):
fname_bkp = '%s-%s' % (fname, gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S'))
os.system('cp %s %s' % (fname, fname_bkp))
if pbits & 1:
print('Existing file %s\nis backed-up %s' % (fname, fname_bkp))
# extend dictionary for other parameters
d = dict(dcmts)
d['run'] = run_start
d['fname'] = os.path.basename(fname)
d['src'] = src
d['ctype'] = type
# make list of comments
cmts = ['%s %s' % (k.upper().ljust(11), v) for k, v in d.iteritems()]
# save n-dimensional numpy array in the tmp text file
fntmp = tempfile.NamedTemporaryFile(mode='r+b', suffix='.data')
if pbits & 2: print('Save constants in tmp file: %s' % fntmp.name)
save_txt(fntmp.name, arr, cmts, fmt='%.1f')
if pbits & 1: print('Deploy constants in file: %s' % fname)
# USE cat in stead of cp and move in order to create output file with correct ACL permissions
cmd_cat = 'cat %s > %s' % (fntmp.name, fname)
#os.system(cmd_cat)
stream = os.popen(cmd_cat)
resp = stream.read()
msg = 'Command: %s\n - resp: %s' % (cmd_cat, resp)
if pbits & 2: print(msg)
# add record to the HISTORY file
hrec = _history_record(d)
if pbits & 1: print('Add record: %sto the file: %s' % (hrec, path_history))
gu.save_textfile(hrec, path_history, mode='a')
def msg_to_log(runs=[]):
"""Returns (str) message to the log file for list of (str) runs.
"""
if len(runs) == 0: return None
tstamp = gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time())
login = gu.get_login()
cwd = gu.get_cwd()
host = gu.get_hostname()
cmd = sys.argv[0].split('/')[-1]
recs = [
'%s %s %s %s cwd:%s cmd:%s' % (s, tstamp, login, host, cwd, cmd)
for s in runs
]
text = '\n'.join(recs)
return text + '\n'
def test_insert_constants(expname=TEST_EXPNAME, detname=TEST_DETNAME, ctype='test_ctype', runnum=10, data='test text sampele'):
""" Inserts constants using direct MongoDB interface from MDBUtils.
"""
import psana.pyalgos.generic.Utils as gu
print('test_delete_database 1:', database_names())
#txt = '%s\nThis is a string\n to test\ncalibration storage' % gu.str_tstamp()
#data, ctype = txt, 'testtext'; logger.debug('txt: %s' % str(data))
#data, ctype = get_test_nda(), 'testnda'; logger.debug(info_ndarr(data, 'nda'))
#data, ctype = get_test_dic(), 'testdict'; logger.debug('dict: %s' % str(data))
kwa = {'user' : gu.get_login()}
t0_sec = time()
ts = gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S%z', time_sec=t0_sec)
mu.insert_constants('%s - saved at %s'%(data,ts), expname, detname, ctype, runnum+int(tname), int(t0_sec),\
time_stamp=ts, **kwa)
print('test_delete_database 2:', database_names())
def deploy_calib_file(cdir,
src,
type,
run_start,
run_end=None,
ifname='',
dcmts={},
pbits=1):
"""Deploys calibration file
- makes the new file name using make_calib_file_name(...)
- if file with this name already exists - rename it with current timestamp in the name
- save array in file
- add history record
"""
fname = make_calib_file_name(cdir, src, type, run_start, run_end, pbits)
path_history = '%s/HISTORY' % os.path.dirname(fname)
if os.path.exists(fname):
fname_bkp = '%s-%s' % (fname, gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S'))
os.system('cp %s %s' % (fname, fname_bkp))
if pbits & 1:
print('Existing file %s\nis backed-up %s' % (fname, fname_bkp))
# extend dictionary for other parameters
d = dict(dcmts)
d['run'] = run_start
d['fname'] = os.path.basename(fname)
d['ifname'] = ifname
d['src'] = src
d['ctype'] = type
if pbits & 1: print('Deploy constants in file: %s' % fname)
# USE cat in stead of cp and move in order to create output file with correct ACL permissions
cmd_cat = 'cat %s > %s' % (ifname, fname)
#os.system(cmd_cat)
stream = os.popen(cmd_cat)
resp = stream.read()
msg = 'Command: %s\n - resp: %s' % (cmd_cat, resp)
if pbits & 2: print(msg)
# add record to the HISTORY file
hrec = _history_record(d)
if pbits & 1: print('Add record: %sto the file: %s' % (hrec, path_history))
gu.save_textfile(hrec, path_history, mode='a')
def do_work():
prefix = './%s-figs-ti_vs_tj' % gu.str_tstamp(
fmt='%Y-%m-%d', time_sec=None) # '%Y-%m-%dT%H:%M:%S%z'
gu.create_directory(prefix, mode=0o775)
path = os.path.abspath(os.path.dirname(__file__))
print('path to npy flies dir:', path)
ti_vs_tj = np.load('%s/ti_vs_tj.npy' % path)
t_all = np.load('%s/t_all.npy' % path)
trange = (1400., 2900.)
print_ndarr(ti_vs_tj, 'ti_vs_tj:\n')
print_ndarr(t_all, 't_all:\n')
sum_bkg = t_all.sum()
sum_cor = ti_vs_tj.sum()
print('sum_bkg:', sum_bkg)
print('sum_cor:', sum_cor)
imrange = trange + trange # (1400., 2900., 1400., 2900.)
axim = gr.plotImageLarge(ti_vs_tj, img_range=imrange, amp_range=(0,500), figsize=(11,10),\
title='ti_vs_tj', origin='lower', window=(0.10, 0.08, 0.88, 0.88), cmap='inferno')
gr.save('%s/fig-ti_vs_tj.png' % prefix)
bkg = np.outer(t_all, t_all) / sum_bkg
print_ndarr(bkg, 'bkg:\n')
axim = gr.plotImageLarge(bkg, img_range=imrange, amp_range=(0,500), figsize=(11,10),\
title='bkg', origin='lower', window=(0.10, 0.08, 0.88, 0.88), cmap='inferno')
gr.save('%s/fig-ti_vs_tj-bkg.png' % prefix)
harr = t_all
nbins = harr.size
ht = HBins(trange, nbins, vtype=np.float32) # ht.binedges()
fig, axhi, hi = gr.hist1d(ht.bincenters(), bins=nbins, amp_range=ht.limits(), weights=harr, color='b', show_stat=True,\
log=True, figsize=(7,6), axwin=(0.10, 0.10, 0.88, 0.85), title='1-d bkg',\
xlabel='time of all hits (ns)', ylabel='number of hits', titwin='1-d bkg')
gr.save('%s/fig-time-hits.png' % prefix)
gr.show()
def _history_record(dcmts):
"""Returns history record made of dictionary comments and system info
"""
user = gu.get_login()
host = gu.get_hostname()
tstamp = gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S zone:%Z')
rnum = '%04d' % dcmts.get('run')
exp = '%s' % dcmts.get('exp')
ifname = '%s' % dcmts.get('ifname')
ofname = '%s' % dcmts.get('fname')
app = '%s' % dcmts.get('app')
cmt = '%s' % dcmts.get('comment')
return 'file:%s copy_of:%s exp:%s run:%s app:%s user:%s host:%s cptime:%s comment:%s\n' % \
(ofname.ljust(14),
ifname,
exp.ljust(8),
rnum.ljust(4),
app.ljust(10),
user,
host,
tstamp.ljust(29),
cmt)
def time_and_timestamp(**kwargs):
"""Returns "time_sec" and "time_stamp" from **kwargs.
If one of these parameters is missing, another is reconstructed from available one.
If both missing - current time is used.
"""
time_sec = kwargs.get('time_sec', None)
time_stamp = kwargs.get('time_stamp', None)
if time_sec is not None:
assert isinstance(
time_sec,
str), 'time_and_timestamp - parameter time_sec should be str'
int_time_sec = int(time_sec)
assert 0 < int_time_sec < 5000000000, 'time_and_timestamp - parameter time_sec should be in allowed range'
if time_stamp is None:
time_stamp = gu.str_tstamp(TSFORMAT, int_time_sec)
else:
if time_stamp is None:
time_sec, time_stamp = gu.time_and_stamp(TSFORMAT)
else:
time_sec = gu.time_sec_from_stamp(TSFORMAT, time_stamp)
return str(time_sec), time_stamp
def _timestamp(time_sec) -> str:
"""Converts time_sec in timestamp of adopted format TSFORMAT.
"""
return gu.str_tstamp(TSFORMAT, int(time_sec))
def get_test_txt():
"""Returns text for test purpose.
"""
return '%s\nThis is a string\n to test\ncalibration storage' % gu.str_tstamp(
)
def proc_control(parser):
"""Dataset processing module
- Submits job in batch for data processing
- Checks in loop batch job status
- Marks in DB that job is processed
- Save common log file for submission and processing
"""
(popts, pargs) = parser.parse_args()
exp = popts.exp
run = popts.run
procname = popts.pro
qname = popts.que
dt_sec = popts.dts
sources = popts.srs
tstamp = gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time())
logpref = rpu.log_batch_prefix(exp, run, procname)
logfname = '%s-%s.txt' % (logpref, tstamp)
for i in range(10):
if gu.create_path(logfname, depth=6, mode=0774, verb=False): continue
gu.save_textfile('\nCreated path: %s' % logfname, logfname, mode='a')
os.chmod(logfname, 0664)
ofprefix = '%s/nda-#exp-#run-#src-#evts-#type.txt' % rpu.work_dir(
exp, procname)
gu.save_textfile('\nOutput work files: %s' % ofprefix, logfname, mode='a')
for i in range(5):
if gu.create_path(ofprefix, depth=8, mode=0774, verb=False): continue
msg = '\nproc_control exp: %s run: %s procname: %s qname: %s logfname %s'%\
(exp, str(run), procname, qname, logfname)
gu.save_textfile(str(msg), logfname, mode='a')
cmd = _batch_submit_command(exp, run, procname, qname, logfname, ofprefix,
sources)
if cmd is None: raise IOError('ERROR: batch submission command is None...')
t0_sec = time()
out, err, jobid = spu.batch_job_submit(cmd, env=None, shell=False)
msg = 'bsub subproc time dt=%7.3f sec' % (time() - t0_sec)
gu.save_textfile(msg, logfname, mode='a')
#if 'submitted without an AFS token' in err :
# print ' Tip: to get rid of error message use commands: kinit; aklog'
rec = gu.log_rec_on_start()
msg = '%s\nSubmitted batch job %s to %s\n cmd: %s\n out: %s\n err: "%s"\n%s\n'%\
(rec, jobid, qname, cmd, out, err.strip('\n'), 80*'_')
#print msg
gu.save_textfile(msg, logfname, mode='a')
# check batch job status in loop
status = None
counter = 0
while status in (None, 'RUN', 'PEND'):
counter += 1
out, err, status = spu.batch_job_status(jobid, user=None, qname=qname)
ts = gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time())
msg = '%4d %s %s job %s status %s' % (counter, ts, qname, jobid,
status)
#print msg
gu.save_textfile('%s\n' % msg, logfname, mode='a')
if status in ('EXIT', 'DONE'): break
sleep(dt_sec)
# change log file name in case of bad status
if status != 'DONE':
logfname_bad = '%s-%s' % (logfname, str(status))
#cmd = 'mv %s %s' % (logfname, logfname_bad)
#os.system(cmd)
os.rename(logfname, logfname_bad)