def config_logger(self, log_fname='log.txt'):
self.append_qwlogger('Start logger\nLog file: %s' % log_fname)
levname = self.log_level.value()
level = self.dict_name_to_level[levname] # e.g. logging.DEBUG
tsfmt = '%Y-%m-%dT%H:%M:%S'
fmt = '%(levelname)s %(name)s: %(message)s' if level==logging.DEBUG else\
'%(asctime)s %(levelname)s %(name)s: %(message)s'
#sys.stdout = sys.stderr = open('/dev/null', 'w')
self.formatter = logging.Formatter(fmt, datefmt=tsfmt)
#logger.addFilter(QWFilter(self)) # register self for callback from filter
# TRICK: add filter to handler to intercept ALL messages
if self.save_log_at_exit:
depth = 6 if log_fname[0] == '/' else 1
gu.create_path(log_fname, depth, mode=0o0777)
self.handler = logging.FileHandler(log_fname, 'w')
else:
self.handler = logging.StreamHandler()
self.handler.addFilter(QWFilter(self))
#self.handler.setLevel(logging.NOTSET) # level
self.handler.setFormatter(self.formatter)
logger.addHandler(self.handler)
self.set_level(levname) # pass level name
def __init__(self, cp, show_buttons=True):
QWidget.__init__(self, parent=None)
self.log_level = cp.log_level
self.log_prefix = cp.log_prefix
self.log_file = cp.log_file # DEPRICATED
log_fname = log_file_name(self.log_prefix.value())
depth = 6 if log_fname[0] == '/' else 1
gu.create_path(log_fname, depth, mode=0o0777)
#print('Log file: %s' % log_fname)
self.show_buttons = show_buttons
cp.qwloggerstd = self
#logger.debug('logging.DEBUG: ', logging.DEBUG)
logger.debug('logging._levelToName: ', logging._levelToName
) # {0: 'NOTSET', 50: 'CRITICAL', 20: 'INFO',...
logger.debug('logging._nameToLevel: ', logging._nameToLevel
) # {'NOTSET': 0, 'ERROR': 40, 'WARNING': 30,...
self.dict_level_to_name = logging._levelToName
self.dict_name_to_level = logging._nameToLevel
self.level_names = list(logging._levelToName.values())
self.edi_txt = QTextEdit('Logger window')
self.lab_level = QLabel('Log level:')
self.but_close = QPushButton('&Close')
self.but_save = QPushButton('&Save log-file')
self.but_rand = QPushButton('&Random')
self.cmb_level = QComboBox(self)
self.cmb_level.addItems(self.level_names)
self.cmb_level.setCurrentIndex(
self.level_names.index(self.log_level.value()))
self.hboxM = QHBoxLayout()
self.hboxM.addWidget(self.edi_txt)
self.hboxB = QHBoxLayout()
self.hboxB.addStretch(4)
self.hboxB.addWidget(self.lab_level)
self.hboxB.addWidget(self.cmb_level)
self.hboxB.addWidget(self.but_rand)
self.hboxB.addStretch(1)
self.hboxB.addWidget(self.but_save)
self.hboxB.addWidget(self.but_close)
self.vbox = QVBoxLayout()
self.vbox.addLayout(self.hboxM)
self.vbox.addLayout(self.hboxB)
self.setLayout(self.vbox)
if self.show_buttons: self.connect_buttons()
self.set_style()
self.set_tool_tips()
self.config_logger(log_fname)
def append_log_file(exp='xpptut15', procname='pixel_status', runs=[], verb=0):
"""Appends records in the log file for list of (str) runs for specified experiment and process name.
"""
fname_log = log_file(exp, procname)
if verb: print('Append log file: %s' % fname_log)
gu.create_path(fname_log, depth=6, mode=0o774, verb=False)
text = msg_to_log(runs)
if text is None: return
#print('Save in file text "%s"' % text)
gu.save_textfile(text, fname_log, mode='a')
os.chmod(fname_log, 0o664)
def proc_control(parser):
"""Dataset processing module
- Submits job in batch for data processing
- Checks in loop batch job status
- Marks in DB that job is processed
- Save common log file for submission and processing
"""
(popts, pargs) = parser.parse_args()
exp = popts.exp
run = popts.run
procname = popts.pro
qname = popts.que
dt_sec = popts.dts
sources = popts.srs
tstamp = gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time())
logpref = rpu.log_batch_prefix(exp, run, procname)
logfname = '%s-%s.txt' % (logpref, tstamp)
for i in range(10):
if gu.create_path(logfname, depth=6, mode=0774, verb=False): continue
gu.save_textfile('\nCreated path: %s' % logfname, logfname, mode='a')
os.chmod(logfname, 0664)
ofprefix = '%s/nda-#exp-#run-#src-#evts-#type.txt' % rpu.work_dir(
exp, procname)
gu.save_textfile('\nOutput work files: %s' % ofprefix, logfname, mode='a')
for i in range(5):
if gu.create_path(ofprefix, depth=8, mode=0774, verb=False): continue
msg = '\nproc_control exp: %s run: %s procname: %s qname: %s logfname %s'%\
(exp, str(run), procname, qname, logfname)
gu.save_textfile(str(msg), logfname, mode='a')
cmd = _batch_submit_command(exp, run, procname, qname, logfname, ofprefix,
sources)
if cmd is None: raise IOError('ERROR: batch submission command is None...')
t0_sec = time()
out, err, jobid = spu.batch_job_submit(cmd, env=None, shell=False)
msg = 'bsub subproc time dt=%7.3f sec' % (time() - t0_sec)
gu.save_textfile(msg, logfname, mode='a')
#if 'submitted without an AFS token' in err :
# print ' Tip: to get rid of error message use commands: kinit; aklog'
rec = gu.log_rec_on_start()
msg = '%s\nSubmitted batch job %s to %s\n cmd: %s\n out: %s\n err: "%s"\n%s\n'%\
(rec, jobid, qname, cmd, out, err.strip('\n'), 80*'_')
#print msg
gu.save_textfile(msg, logfname, mode='a')
# check batch job status in loop
status = None
counter = 0
while status in (None, 'RUN', 'PEND'):
counter += 1
out, err, status = spu.batch_job_status(jobid, user=None, qname=qname)
ts = gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time())
msg = '%4d %s %s job %s status %s' % (counter, ts, qname, jobid,
status)
#print msg
gu.save_textfile('%s\n' % msg, logfname, mode='a')
if status in ('EXIT', 'DONE'): break
sleep(dt_sec)
# change log file name in case of bad status
if status != 'DONE':
logfname_bad = '%s-%s' % (logfname, str(status))
#cmd = 'mv %s %s' % (logfname, logfname_bad)
#os.system(cmd)
os.rename(logfname, logfname_bad)