def config_logger(self, log_fname='log.txt'): self.append_qwlogger('Start logger\nLog file: %s' % log_fname) levname = self.log_level.value() level = self.dict_name_to_level[levname] # e.g. logging.DEBUG tsfmt = '%Y-%m-%dT%H:%M:%S' fmt = '%(levelname)s %(name)s: %(message)s' if level==logging.DEBUG else\ '%(asctime)s %(levelname)s %(name)s: %(message)s' #sys.stdout = sys.stderr = open('/dev/null', 'w') self.formatter = logging.Formatter(fmt, datefmt=tsfmt) #logger.addFilter(QWFilter(self)) # register self for callback from filter # TRICK: add filter to handler to intercept ALL messages if self.save_log_at_exit: depth = 6 if log_fname[0] == '/' else 1 gu.create_path(log_fname, depth, mode=0o0777) self.handler = logging.FileHandler(log_fname, 'w') else: self.handler = logging.StreamHandler() self.handler.addFilter(QWFilter(self)) #self.handler.setLevel(logging.NOTSET) # level self.handler.setFormatter(self.formatter) logger.addHandler(self.handler) self.set_level(levname) # pass level name
def __init__(self, cp, show_buttons=True): QWidget.__init__(self, parent=None) self.log_level = cp.log_level self.log_prefix = cp.log_prefix self.log_file = cp.log_file # DEPRICATED log_fname = log_file_name(self.log_prefix.value()) depth = 6 if log_fname[0] == '/' else 1 gu.create_path(log_fname, depth, mode=0o0777) #print('Log file: %s' % log_fname) self.show_buttons = show_buttons cp.qwloggerstd = self #logger.debug('logging.DEBUG: ', logging.DEBUG) logger.debug('logging._levelToName: ', logging._levelToName ) # {0: 'NOTSET', 50: 'CRITICAL', 20: 'INFO',... logger.debug('logging._nameToLevel: ', logging._nameToLevel ) # {'NOTSET': 0, 'ERROR': 40, 'WARNING': 30,... self.dict_level_to_name = logging._levelToName self.dict_name_to_level = logging._nameToLevel self.level_names = list(logging._levelToName.values()) self.edi_txt = QTextEdit('Logger window') self.lab_level = QLabel('Log level:') self.but_close = QPushButton('&Close') self.but_save = QPushButton('&Save log-file') self.but_rand = QPushButton('&Random') self.cmb_level = QComboBox(self) self.cmb_level.addItems(self.level_names) self.cmb_level.setCurrentIndex( self.level_names.index(self.log_level.value())) self.hboxM = QHBoxLayout() self.hboxM.addWidget(self.edi_txt) self.hboxB = QHBoxLayout() self.hboxB.addStretch(4) self.hboxB.addWidget(self.lab_level) self.hboxB.addWidget(self.cmb_level) self.hboxB.addWidget(self.but_rand) self.hboxB.addStretch(1) self.hboxB.addWidget(self.but_save) self.hboxB.addWidget(self.but_close) self.vbox = QVBoxLayout() self.vbox.addLayout(self.hboxM) self.vbox.addLayout(self.hboxB) self.setLayout(self.vbox) if self.show_buttons: self.connect_buttons() self.set_style() self.set_tool_tips() self.config_logger(log_fname)
def append_log_file(exp='xpptut15', procname='pixel_status', runs=[], verb=0): """Appends records in the log file for list of (str) runs for specified experiment and process name. """ fname_log = log_file(exp, procname) if verb: print('Append log file: %s' % fname_log) gu.create_path(fname_log, depth=6, mode=0o774, verb=False) text = msg_to_log(runs) if text is None: return #print('Save in file text "%s"' % text) gu.save_textfile(text, fname_log, mode='a') os.chmod(fname_log, 0o664)
def proc_control(parser): """Dataset processing module - Submits job in batch for data processing - Checks in loop batch job status - Marks in DB that job is processed - Save common log file for submission and processing """ (popts, pargs) = parser.parse_args() exp = popts.exp run = popts.run procname = popts.pro qname = popts.que dt_sec = popts.dts sources = popts.srs tstamp = gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time()) logpref = rpu.log_batch_prefix(exp, run, procname) logfname = '%s-%s.txt' % (logpref, tstamp) for i in range(10): if gu.create_path(logfname, depth=6, mode=0774, verb=False): continue gu.save_textfile('\nCreated path: %s' % logfname, logfname, mode='a') os.chmod(logfname, 0664) ofprefix = '%s/nda-#exp-#run-#src-#evts-#type.txt' % rpu.work_dir( exp, procname) gu.save_textfile('\nOutput work files: %s' % ofprefix, logfname, mode='a') for i in range(5): if gu.create_path(ofprefix, depth=8, mode=0774, verb=False): continue msg = '\nproc_control exp: %s run: %s procname: %s qname: %s logfname %s'%\ (exp, str(run), procname, qname, logfname) gu.save_textfile(str(msg), logfname, mode='a') cmd = _batch_submit_command(exp, run, procname, qname, logfname, ofprefix, sources) if cmd is None: raise IOError('ERROR: batch submission command is None...') t0_sec = time() out, err, jobid = spu.batch_job_submit(cmd, env=None, shell=False) msg = 'bsub subproc time dt=%7.3f sec' % (time() - t0_sec) gu.save_textfile(msg, logfname, mode='a') #if 'submitted without an AFS token' in err : # print ' Tip: to get rid of error message use commands: kinit; aklog' rec = gu.log_rec_on_start() msg = '%s\nSubmitted batch job %s to %s\n cmd: %s\n out: %s\n err: "%s"\n%s\n'%\ (rec, jobid, qname, cmd, out, err.strip('\n'), 80*'_') #print msg gu.save_textfile(msg, logfname, mode='a') # check batch job status in loop status = None counter = 0 while status in (None, 'RUN', 'PEND'): counter += 1 out, err, status = spu.batch_job_status(jobid, user=None, qname=qname) ts = gu.str_tstamp('%Y-%m-%dT%H:%M:%S', time()) msg = '%4d %s %s job %s status %s' % (counter, ts, qname, jobid, status) #print msg gu.save_textfile('%s\n' % msg, logfname, mode='a') if status in ('EXIT', 'DONE'): break sleep(dt_sec) # change log file name in case of bad status if status != 'DONE': logfname_bad = '%s-%s' % (logfname, str(status)) #cmd = 'mv %s %s' % (logfname, logfname_bad) #os.system(cmd) os.rename(logfname, logfname_bad)