def move_recs_to_archive(procname, exp, runs):
"""Move expired run records from log file to archive file.
"""
fname_log = log_file(exp, procname)
fname_arc = arc_file(exp, procname)
print('Move records for old runs to archive file: %s\n' % fname_arc)
recs = gu.load_textfile(fname_log).split('\n')
recs_log = [rec for rec in recs if not (rec[:4] in runs)]
recs_arc = [rec for rec in recs if rec[:4] in runs]
text_log = '\n'.join(recs_log)
text_log += '\n'
#if len(runs)==1 : text_log+='\n'
text_arc = '\n'.join(recs_arc)
text_arc += '\n'
#if len(runs)==1 : text_arc+='\n'
#print(' ==> log\n%s' % text_log)
#print(' ==> arc\n%s' % text_arc)
gu.save_textfile(text_log, fname_log, mode='w')
os.chmod(fname_log, 0o664)
gu.save_textfile(text_arc, fname_arc, mode='a')
os.chmod(fname_log, 0o664)
def append_log_file(exp='xpptut15', procname='pixel_status', runs=[], verb=0):
"""Appends records in the log file for list of (str) runs for specified experiment and process name.
"""
fname_log = log_file(exp, procname)
if verb: print('Append log file: %s' % fname_log)
gu.create_path(fname_log, depth=6, mode=0o774, verb=False)
text = msg_to_log(runs)
if text is None: return
#print('Save in file text "%s"' % text)
gu.save_textfile(text, fname_log, mode='a')
os.chmod(fname_log, 0o664)
def deploy_calib_array(cdir,
src,
type,
run_start,
run_end=None,
arr=None,
dcmts={},
fmt='%.1f',
pbits=1):
"""Deploys array in calibration file
- makes the new file name using make_calib_file_name(...)
- if file with this name already exists - rename it with current timestamp in the name
- save array in file
- add history record
"""
fname = make_calib_file_name(cdir, src, type, run_start, run_end, pbits)
path_history = '%s/HISTORY' % os.path.dirname(fname)
if os.path.exists(fname):
fname_bkp = '%s-%s' % (fname, gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S'))
os.system('cp %s %s' % (fname, fname_bkp))
if pbits & 1:
print('Existing file %s\nis backed-up %s' % (fname, fname_bkp))
# extend dictionary for other parameters
d = dict(dcmts)
d['run'] = run_start
d['fname'] = os.path.basename(fname)
d['src'] = src
d['ctype'] = type
# make list of comments
cmts = ['%s %s' % (k.upper().ljust(11), v) for k, v in d.iteritems()]
# save n-dimensional numpy array in the tmp text file
fntmp = tempfile.NamedTemporaryFile(mode='r+b', suffix='.data')
if pbits & 2: print('Save constants in tmp file: %s' % fntmp.name)
save_txt(fntmp.name, arr, cmts, fmt='%.1f')
if pbits & 1: print('Deploy constants in file: %s' % fname)
# USE cat in stead of cp and move in order to create output file with correct ACL permissions
cmd_cat = 'cat %s > %s' % (fntmp.name, fname)
#os.system(cmd_cat)
stream = os.popen(cmd_cat)
resp = stream.read()
msg = 'Command: %s\n - resp: %s' % (cmd_cat, resp)
if pbits & 2: print(msg)
# add record to the HISTORY file
hrec = _history_record(d)
if pbits & 1: print('Add record: %sto the file: %s' % (hrec, path_history))
gu.save_textfile(hrec, path_history, mode='a')
def deploy_calib_file(cdir,
src,
type,
run_start,
run_end=None,
ifname='',
dcmts={},
pbits=1):
"""Deploys calibration file
- makes the new file name using make_calib_file_name(...)
- if file with this name already exists - rename it with current timestamp in the name
- save array in file
- add history record
"""
fname = make_calib_file_name(cdir, src, type, run_start, run_end, pbits)
path_history = '%s/HISTORY' % os.path.dirname(fname)
if os.path.exists(fname):
fname_bkp = '%s-%s' % (fname, gu.str_tstamp(fmt='%Y-%m-%dT%H:%M:%S'))
os.system('cp %s %s' % (fname, fname_bkp))
if pbits & 1:
print('Existing file %s\nis backed-up %s' % (fname, fname_bkp))
# extend dictionary for other parameters
d = dict(dcmts)
d['run'] = run_start
d['fname'] = os.path.basename(fname)
d['ifname'] = ifname
d['src'] = src
d['ctype'] = type
if pbits & 1: print('Deploy constants in file: %s' % fname)
# USE cat in stead of cp and move in order to create output file with correct ACL permissions
cmd_cat = 'cat %s > %s' % (ifname, fname)
#os.system(cmd_cat)
stream = os.popen(cmd_cat)
resp = stream.read()
msg = 'Command: %s\n - resp: %s' % (cmd_cat, resp)
if pbits & 2: print(msg)
# add record to the HISTORY file
hrec = _history_record(d)
if pbits & 1: print('Add record: %sto the file: %s' % (hrec, path_history))
gu.save_textfile(hrec, path_history, mode='a')
def get(self):
"""Saves in file calibration constants from database.
"""
mode, kwargs = self.mode, self.kwargs
defs = self.defs
host = kwargs.get('host', None)
port = kwargs.get('port', None)
exp = kwargs.get('experiment', None)
det = kwargs.get('detector', None)
ctype = kwargs.get('ctype', None)
run = kwargs.get('run', None)
run_end = kwargs.get('run_end', None)
tsec = kwargs.get('time_sec', None)
tstamp = kwargs.get('time_stamp', None)
vers = kwargs.get('version', None)
fname = kwargs.get('iofname', None)
query = {'detector': det, 'ctype': ctype}
if run != defs['run']:
query['run'] = {'$lte': run}
query['run_end'] = {'$gte': run}
if tsec != defs['time_sec']: query['time_sec'] = {'$lte': tsec}
if vers != defs['version']: query['version'] = vers
#logger.debug('query: %s' % str(query))
dbname = dbu.get_dbname(**kwargs)
client = self.client()
if not self.check_database(client, dbname): return
detname = kwargs.get('detector', None)
if detname is None:
logger.warning(
'%s needs in the collection name. Please specify the detector name.'
% (mode))
colname = detname
db, fs = dbu.db_and_fs(client, dbname)
colnames = dbu.collection_names(db)
if not (colname in colnames): # dbu.collection_exists(db, colname)
logger.warning('db "%s" does not have collection "%s"' %
(db.name, str(colname)))
return
col = dbu.collection(db, colname)
logger.info('Search document in db "%s" collection "%s"' %
(dbname, colname))
#client, expname, detname, db_exp, db_det, fs_exp, fs_det, col_exp, col_det =\
# dbu.connect(host=host, port=port, experiment=exp, detector=det, verbose=verb)
#fs, doc = fs_exp, dbu.find_doc(col, query)
#if doc is None :
# fs, doc = fs_det, dbu.find_doc(col_det, query)
# if doc is None :
# logger.warning('Can not find document for query: %s' % str(query))
# return
doc = dbu.find_doc(col, query)
if doc is None:
logger.warning('Can not find document for query: %s' % str(query))
return
data = dbu.get_data_for_doc(fs, doc)
if data is None:
logger.warning('Can not load data for doc: %s' % str(doc))
return
if fname is None: fname = 'clb-%s-%s-%s.npy' % (expname, det, ctype)
if ctype == 'geometry':
gu.save_textfile(data, fname, mode='w', verb=verb)
elif os.path.splitext(fname)[1] == '.npy':
np.save(fname, data, allow_pickle=False)
else:
save_txt(fname, data, fmt='%.2f')
if verb: logger.info(info_ndarr(data, 'nda', first=0, last=3))
logger.info('Save constants in file: %s' % fname)
def proc_control(parser):
"""Dataset processing module
- Submits job in batch for data processing
- Checks in loop batch job status
- Marks in DB that job is processed
- Save common log file for submission and processing
"""
(popts, pargs) = parser.parse_args()
exp = popts.exp
run = popts.run
procname = popts.pro
qname = popts.que
dt_sec = popts.dts
sources = popts.srs
tstamp = gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time())
logpref = rpu.log_batch_prefix(exp, run, procname)
logfname = '%s-%s.txt' % (logpref, tstamp)
for i in range(10):
if gu.create_path(logfname, depth=6, mode=0774, verb=False): continue
gu.save_textfile('\nCreated path: %s' % logfname, logfname, mode='a')
os.chmod(logfname, 0664)
ofprefix = '%s/nda-#exp-#run-#src-#evts-#type.txt' % rpu.work_dir(
exp, procname)
gu.save_textfile('\nOutput work files: %s' % ofprefix, logfname, mode='a')
for i in range(5):
if gu.create_path(ofprefix, depth=8, mode=0774, verb=False): continue
msg = '\nproc_control exp: %s run: %s procname: %s qname: %s logfname %s'%\
(exp, str(run), procname, qname, logfname)
gu.save_textfile(str(msg), logfname, mode='a')
cmd = _batch_submit_command(exp, run, procname, qname, logfname, ofprefix,
sources)
if cmd is None: raise IOError('ERROR: batch submission command is None...')
t0_sec = time()
out, err, jobid = spu.batch_job_submit(cmd, env=None, shell=False)
msg = 'bsub subproc time dt=%7.3f sec' % (time() - t0_sec)
gu.save_textfile(msg, logfname, mode='a')
#if 'submitted without an AFS token' in err :
# print ' Tip: to get rid of error message use commands: kinit; aklog'
rec = gu.log_rec_on_start()
msg = '%s\nSubmitted batch job %s to %s\n cmd: %s\n out: %s\n err: "%s"\n%s\n'%\
(rec, jobid, qname, cmd, out, err.strip('\n'), 80*'_')
#print msg
gu.save_textfile(msg, logfname, mode='a')
# check batch job status in loop
status = None
counter = 0
while status in (None, 'RUN', 'PEND'):
counter += 1
out, err, status = spu.batch_job_status(jobid, user=None, qname=qname)
ts = gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time())
msg = '%4d %s %s job %s status %s' % (counter, ts, qname, jobid,
status)
#print msg
gu.save_textfile('%s\n' % msg, logfname, mode='a')
if status in ('EXIT', 'DONE'): break
sleep(dt_sec)
# change log file name in case of bad status
if status != 'DONE':
logfname_bad = '%s-%s' % (logfname, str(status))
#cmd = 'mv %s %s' % (logfname, logfname_bad)
#os.system(cmd)
os.rename(logfname, logfname_bad)