class CustomLogHandler(logging.Handler): """multiprocessing log handler This handler makes it possible for several processes to log to the same file by using a queue. """ def __init__(self, fname): logging.Handler.__init__(self) timestamp = str(datetime.now().strftime('%Y%m%dT_%H%M%S')) self._handler = FH("logs/{0}-{1}.log".format(fname, timestamp)) self.queue = multiprocessing.Queue(-1) thrd = threading.Thread(target=self.receive) thrd.daemon = True thrd.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
def close(self): if self.closed: return self.closed = True if self.message_count > self.flood_level: hidden = self.message_count - self.flood_level - len(self.tail) if hidden: # send critical error FileHandler.emit(self, LogRecord( name='flood', level=CRITICAL, pathname='', lineno=0, msg=flood_template % ( self.message_count - self.flood_level - len(self.tail), self.flood_level ), args=(), exc_info=None )) for record in self.tail: FileHandler.emit(self, record) FileHandler.close(self) if PY2: f = os.fdopen(self.fd) summary = f.read().decode(self.charset) else: f = open(self.fd, encoding=self.charset) summary = f.read() f.close() try: encoded_summary = summary.encode('ascii') self.mailer.charset = 'ascii' except UnicodeEncodeError: pass else: if PY2: summary = encoded_summary if os.path.exists(self.filename): os.remove(self.filename) if self.send_level is None or self.maxlevelno >= self.send_level: self.mailer.handle( LogRecord( name='Summary', level=self.maxlevelno, pathname='', lineno=0, msg=summary, args=(), exc_info=None ) )
def emit(self, record): try: if self.check_base_filename(): self.build_base_filename() FileHandler.emit(self, record) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record)
def emit(self, record): now = datetime.datetime.now() if now > self._rotateAt: # time to rotate self._rotateAt = self._getNextRotateTime() self.close() self._tryCleanFiles() FileHandler.emit(self, record)
class MultiProcessingLog(logging.Handler): def __init__(self, name): logging.Handler.__init__(self) self._handler = FileHandler(name) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
def emit(self, record): if self.closed: return for criterion in self.ignore: if criterion(record.msg): return if record.levelno > self.maxlevelno: self.maxlevelno = record.levelno FileHandler.emit(self, record)
def emit(self, record): try: if self.should_rollover(record): self.do_rollover() if self.should_skip(record): return FileHandler.emit(self, record) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record)
def emit(self,record): if self.closed: return for criterion in self.ignore: if criterion(record.msg): return if record.levelno>self.maxlevelno: self.maxlevelno = record.levelno FileHandler.emit(self,record)
def emit(self, record): # Create the file stream if not already created. if self.stream is None: self.stream = self._open() # If the file pre_exists, it should already have a header. # Else write the header to the file so that it is the first line. if not self.file_pre_exists: self.stream.write('%s\n' % self.header) # Call the parent class emit function. FileHandler.emit(self, record)
class MPLogHandler(logging.Handler): """a multiprocessing-compatible file log handler - all processes log to the same file""" def __init__(self, fname): logging.Handler.__init__(self) self._handler = FileHandler(fname) self.queue = multiprocessing.Queue(-1) thrd = threading.Thread(target=self.receive) thrd.daemon = True thrd.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: # dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
def close(self): if self.closed: return self.closed = True if self.message_count > self.flood_level: hidden = self.message_count - self.flood_level - len(self.tail) if hidden: # send critical error FileHandler.emit( self, LogRecord(name='flood', level=CRITICAL, pathname='', lineno=0, msg=flood_template % (self.message_count - self.flood_level - len(self.tail), self.flood_level), args=(), exc_info=None)) for record in self.tail: FileHandler.emit(self, record) FileHandler.close(self) if PY2: f = os.fdopen(self.fd) summary = f.read().decode(self.charset) else: f = open(self.fd, encoding=self.charset) summary = f.read() f.close() try: encoded_summary = summary.encode('ascii') self.mailer.charset = 'ascii' except UnicodeEncodeError: pass else: if PY2: summary = encoded_summary if os.path.exists(self.filename): os.remove(self.filename) if self.send_level is None or self.maxlevelno >= self.send_level: self.mailer.handle( LogRecord(name='Summary', level=self.maxlevelno, pathname='', lineno=0, msg=summary, args=(), exc_info=None))
def emit(self, record): """ Emit a record. Always check time """ try: if self.check_baseFilename(record): self.build_baseFilename() FileHandler.emit(self, record) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record)
def emit(self, record): """ Emit a record. Always check time """ try: if self.check_base_filename(record): self.build_base_filename() FileHandler.emit(self, record) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record)
def emit(self, record): if self.closed: return if record.levelno > self.maxlevelno: self.maxlevelno = record.levelno self.message_count += 1 if self.message_count > self.flood_level: if self.tail is None: self.tail = deque(maxlen=5) self.tail.append(record) else: FileHandler.emit(self, record)
def close(self): if self.closed: return if self.message_count > self.flood_level: hidden = self.message_count - self.flood_level - len(self.tail) if hidden: # send critical error FileHandler.emit(self, LogRecord( name='flood', level=CRITICAL, pathname='', lineno=0, msg=flood_template % ( self.message_count - self.flood_level - len(self.tail), self.flood_level ), args=(), exc_info=None )) for record in self.tail: FileHandler.emit(self, record) FileHandler.close(self) f = os.fdopen(self.fd) summary = f.read().decode(self.charset) f.close() # try and encode in ascii, to keep emails simpler: try: summary = summary.encode('ascii') except UnicodeEncodeError: # unicode it is then pass if os.path.exists(self.filename): os.remove(self.filename) if self.send_level is None or self.maxlevelno >= self.send_level: self.mailer.handle( LogRecord( name='Summary', level=self.maxlevelno, pathname='', lineno=0, msg=summary, args=(), exc_info=None ) ) self.closed = True
def emit(self, record): exception_info = record.exc_info record.exc_info = False if isinstance(exception_info, tuple): exception_info = ExceptionInfo.from_values(*exception_info) elif exception_info: exception_info = ExceptionInfo.new() # super() raises exception in Python 2.6 # TypeError: super() argument 1 must be type, not classobj LoggingFileHandler.emit(self, record) if exception_info: data = self.formatter.format_exception(exception_info, record.getMessage()) self._storage.save(data, exception_info)
def emit(self, record): """ Emit a record. Always check time """ try: if self.check_baseFilename(record): self.build_baseFilename() FileHandler.emit(self, record) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) finally: if self.backupCount > 0: to_delete_files = self.getFilesToDelete() for s in to_delete_files: os.remove(s)
def emit(self, record): if not record.levelno == INFO: return FileHandler.emit(self,record)
def emit(self, record): if not record.levelno == WARNING: return FileHandler.emit(self,record)
def emit(self, record): if self.configHash != _ConfigFile.md5sum(): _config() self.configHash = _ConfigFile.md5sum() FileHandler.emit(self, record)
class MultiProcessingLogHandler(logging.Handler): def __init__(self, name, queue): logging.Handler.__init__(self) self._handler = FileHandler(name) self.queue = queue t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() if record == StopIteration: break self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except Exception: traceback.print_exc(file=sys.stderr) break return def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except Exception: self.handleError(record) def close(self): self.queue.put_nowait(StopIteration) self._handler.close() logging.Handler.close(self)
def emit(self, record): self.baseFilename = self._get_current_filename() FileHandler.emit(self, record) self.close()
def emit(self, record): if not record.levelno == logging.DEBUG: return FileHandler.emit(self, record)
#!encoding=utf-8 from logging import FileHandler from logging import LogRecord import sys onelog=LogRecord('name','10',pathname=__file__,lineno=10,msg=u'haha哈',args=None,exc_info=None,func=None) F=FileHandler(filename='my_log') F.emit(onelog)
class LogHandler(logging.Handler): """multiprocessing log handler This handler makes it possible for several processes to log to the same file by using a queue. """ def __init__(self, fname): logging.Handler.__init__(self) self._handler = FH(self._make_filename(fname)) self.queue = multiprocessing.Queue(-1) thrd = threading.Thread(target=self.receive) thrd.daemon = True thrd.start() @classmethod def _make_filename(self, fname): filename = [] filename.append(fname) filename.append('_') unix_timestamp = int(time.time()) filename.append(str(unix_timestamp)) filename.append('.log') return ''.join(filename) def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except Exception: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except Exception: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)