class CustomLogHandler(logging.Handler):
"""multiprocessing log handler
This handler makes it possible for several processes
to log to the same file by using a queue.
"""
def __init__(self, fname):
logging.Handler.__init__(self)
timestamp = str(datetime.now().strftime('%Y%m%dT_%H%M%S'))
self._handler = FH("logs/{0}-{1}.log".format(fname, timestamp))
self.queue = multiprocessing.Queue(-1)
thrd = threading.Thread(target=self.receive)
thrd.daemon = True
thrd.start()
def setFormatter(self, fmt):
logging.Handler.setFormatter(self, fmt)
self._handler.setFormatter(fmt)
def receive(self):
while True:
try:
record = self.queue.get()
self._handler.emit(record)
except (KeyboardInterrupt, SystemExit):
raise
except EOFError:
break
except:
traceback.print_exc(file=sys.stderr)
def send(self, s):
self.queue.put_nowait(s)
def _format_record(self, record):
if record.args:
record.msg = record.msg % record.args
record.args = None
if record.exc_info:
dummy = self.format(record)
record.exc_info = None
return record
def emit(self, record):
try:
s = self._format_record(record)
self.send(s)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def close(self):
self._handler.close()
logging.Handler.close(self)
def close(self):
if self.closed:
return
self.closed = True
if self.message_count > self.flood_level:
hidden = self.message_count - self.flood_level - len(self.tail)
if hidden:
# send critical error
FileHandler.emit(self, LogRecord(
name='flood',
level=CRITICAL,
pathname='',
lineno=0,
msg=flood_template % (
self.message_count - self.flood_level - len(self.tail),
self.flood_level
),
args=(),
exc_info=None
))
for record in self.tail:
FileHandler.emit(self, record)
FileHandler.close(self)
if PY2:
f = os.fdopen(self.fd)
summary = f.read().decode(self.charset)
else:
f = open(self.fd, encoding=self.charset)
summary = f.read()
f.close()
try:
encoded_summary = summary.encode('ascii')
self.mailer.charset = 'ascii'
except UnicodeEncodeError:
pass
else:
if PY2:
summary = encoded_summary
if os.path.exists(self.filename):
os.remove(self.filename)
if self.send_level is None or self.maxlevelno >= self.send_level:
self.mailer.handle(
LogRecord(
name='Summary',
level=self.maxlevelno,
pathname='',
lineno=0,
msg=summary,
args=(),
exc_info=None
)
)
def emit(self, record):
try:
if self.check_base_filename():
self.build_base_filename()
FileHandler.emit(self, record)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def emit(self, record):
now = datetime.datetime.now()
if now > self._rotateAt:
# time to rotate
self._rotateAt = self._getNextRotateTime()
self.close()
self._tryCleanFiles()
FileHandler.emit(self, record)
class MultiProcessingLog(logging.Handler):
def __init__(self, name):
logging.Handler.__init__(self)
self._handler = FileHandler(name)
self.queue = multiprocessing.Queue(-1)
t = threading.Thread(target=self.receive)
t.daemon = True
t.start()
def setFormatter(self, fmt):
logging.Handler.setFormatter(self, fmt)
self._handler.setFormatter(fmt)
def receive(self):
while True:
try:
record = self.queue.get()
self._handler.emit(record)
except (KeyboardInterrupt, SystemExit):
raise
except EOFError:
break
except:
traceback.print_exc(file=sys.stderr)
def send(self, s):
self.queue.put_nowait(s)
def _format_record(self, record):
# ensure that exc_info and args
# have been stringified. Removes any chance of
# unpickleable things inside and possibly reduces
# message size sent over the pipe
if record.args:
record.msg = record.msg % record.args
record.args = None
if record.exc_info:
dummy = self.format(record)
record.exc_info = None
return record
def emit(self, record):
try:
s = self._format_record(record)
self.send(s)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def close(self):
self._handler.close()
logging.Handler.close(self)
def emit(self, record):
if self.closed:
return
for criterion in self.ignore:
if criterion(record.msg):
return
if record.levelno > self.maxlevelno:
self.maxlevelno = record.levelno
FileHandler.emit(self, record)
def emit(self, record):
try:
if self.should_rollover(record):
self.do_rollover()
if self.should_skip(record):
return
FileHandler.emit(self, record)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def emit(self,record):
if self.closed:
return
for criterion in self.ignore:
if criterion(record.msg):
return
if record.levelno>self.maxlevelno:
self.maxlevelno = record.levelno
FileHandler.emit(self,record)
def emit(self, record):
# Create the file stream if not already created.
if self.stream is None:
self.stream = self._open()
# If the file pre_exists, it should already have a header.
# Else write the header to the file so that it is the first line.
if not self.file_pre_exists:
self.stream.write('%s\n' % self.header)
# Call the parent class emit function.
FileHandler.emit(self, record)
class MPLogHandler(logging.Handler):
"""a multiprocessing-compatible file log handler -
all processes log to the same file"""
def __init__(self, fname):
logging.Handler.__init__(self)
self._handler = FileHandler(fname)
self.queue = multiprocessing.Queue(-1)
thrd = threading.Thread(target=self.receive)
thrd.daemon = True
thrd.start()
def setFormatter(self, fmt):
logging.Handler.setFormatter(self, fmt)
self._handler.setFormatter(fmt)
def receive(self):
while True:
try:
record = self.queue.get()
self._handler.emit(record)
except (KeyboardInterrupt, SystemExit):
raise
except EOFError:
break
except:
traceback.print_exc(file=sys.stderr)
def send(self, s):
self.queue.put_nowait(s)
def _format_record(self, record):
if record.args:
record.msg = record.msg % record.args
record.args = None
if record.exc_info:
# dummy = self.format(record)
record.exc_info = None
return record
def emit(self, record):
try:
s = self._format_record(record)
self.send(s)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def close(self):
self._handler.close()
logging.Handler.close(self)
def close(self):
if self.closed:
return
self.closed = True
if self.message_count > self.flood_level:
hidden = self.message_count - self.flood_level - len(self.tail)
if hidden:
# send critical error
FileHandler.emit(
self,
LogRecord(name='flood',
level=CRITICAL,
pathname='',
lineno=0,
msg=flood_template %
(self.message_count - self.flood_level -
len(self.tail), self.flood_level),
args=(),
exc_info=None))
for record in self.tail:
FileHandler.emit(self, record)
FileHandler.close(self)
if PY2:
f = os.fdopen(self.fd)
summary = f.read().decode(self.charset)
else:
f = open(self.fd, encoding=self.charset)
summary = f.read()
f.close()
try:
encoded_summary = summary.encode('ascii')
self.mailer.charset = 'ascii'
except UnicodeEncodeError:
pass
else:
if PY2:
summary = encoded_summary
if os.path.exists(self.filename):
os.remove(self.filename)
if self.send_level is None or self.maxlevelno >= self.send_level:
self.mailer.handle(
LogRecord(name='Summary',
level=self.maxlevelno,
pathname='',
lineno=0,
msg=summary,
args=(),
exc_info=None))
def emit(self, record):
"""
Emit a record.
Always check time
"""
try:
if self.check_baseFilename(record):
self.build_baseFilename()
FileHandler.emit(self, record)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def emit(self, record):
"""
Emit a record.
Always check time
"""
try:
if self.check_base_filename(record):
self.build_base_filename()
FileHandler.emit(self, record)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def emit(self, record):
if self.closed:
return
if record.levelno > self.maxlevelno:
self.maxlevelno = record.levelno
self.message_count += 1
if self.message_count > self.flood_level:
if self.tail is None:
self.tail = deque(maxlen=5)
self.tail.append(record)
else:
FileHandler.emit(self, record)
def close(self):
if self.closed:
return
if self.message_count > self.flood_level:
hidden = self.message_count - self.flood_level - len(self.tail)
if hidden:
# send critical error
FileHandler.emit(self, LogRecord(
name='flood',
level=CRITICAL,
pathname='',
lineno=0,
msg=flood_template % (
self.message_count - self.flood_level - len(self.tail),
self.flood_level
),
args=(),
exc_info=None
))
for record in self.tail:
FileHandler.emit(self, record)
FileHandler.close(self)
f = os.fdopen(self.fd)
summary = f.read().decode(self.charset)
f.close()
# try and encode in ascii, to keep emails simpler:
try:
summary = summary.encode('ascii')
except UnicodeEncodeError:
# unicode it is then
pass
if os.path.exists(self.filename):
os.remove(self.filename)
if self.send_level is None or self.maxlevelno >= self.send_level:
self.mailer.handle(
LogRecord(
name='Summary',
level=self.maxlevelno,
pathname='',
lineno=0,
msg=summary,
args=(),
exc_info=None
)
)
self.closed = True
def emit(self, record):
exception_info = record.exc_info
record.exc_info = False
if isinstance(exception_info, tuple):
exception_info = ExceptionInfo.from_values(*exception_info)
elif exception_info:
exception_info = ExceptionInfo.new()
# super() raises exception in Python 2.6
# TypeError: super() argument 1 must be type, not classobj
LoggingFileHandler.emit(self, record)
if exception_info:
data = self.formatter.format_exception(exception_info,
record.getMessage())
self._storage.save(data, exception_info)
def emit(self, record):
"""
Emit a record.
Always check time
"""
try:
if self.check_baseFilename(record):
self.build_baseFilename()
FileHandler.emit(self, record)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
finally:
if self.backupCount > 0:
to_delete_files = self.getFilesToDelete()
for s in to_delete_files:
os.remove(s)
def emit(self, record):
if not record.levelno == INFO:
return
FileHandler.emit(self,record)
def emit(self, record):
if not record.levelno == WARNING:
return
FileHandler.emit(self,record)
def emit(self, record):
if self.configHash != _ConfigFile.md5sum():
_config()
self.configHash = _ConfigFile.md5sum()
FileHandler.emit(self, record)
class MultiProcessingLogHandler(logging.Handler):
def __init__(self, name, queue):
logging.Handler.__init__(self)
self._handler = FileHandler(name)
self.queue = queue
t = threading.Thread(target=self.receive)
t.daemon = True
t.start()
def setFormatter(self, fmt):
logging.Handler.setFormatter(self, fmt)
self._handler.setFormatter(fmt)
def receive(self):
while True:
try:
record = self.queue.get()
if record == StopIteration:
break
self._handler.emit(record)
except (KeyboardInterrupt, SystemExit):
raise
except EOFError:
break
except Exception:
traceback.print_exc(file=sys.stderr)
break
return
def send(self, s):
self.queue.put_nowait(s)
def _format_record(self, record):
# ensure that exc_info and args
# have been stringified. Removes any chance of
# unpickleable things inside and possibly reduces
# message size sent over the pipe
if record.args:
record.msg = record.msg % record.args
record.args = None
if record.exc_info:
dummy = self.format(record)
record.exc_info = None
return record
def emit(self, record):
try:
s = self._format_record(record)
self.send(s)
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
self.handleError(record)
def close(self):
self.queue.put_nowait(StopIteration)
self._handler.close()
logging.Handler.close(self)
def emit(self, record):
self.baseFilename = self._get_current_filename()
FileHandler.emit(self, record)
self.close()
def emit(self, record):
if not record.levelno == logging.DEBUG:
return
FileHandler.emit(self, record)
#!encoding=utf-8
from logging import FileHandler
from logging import LogRecord
import sys
onelog=LogRecord('name','10',pathname=__file__,lineno=10,msg=u'haha哈',args=None,exc_info=None,func=None)
F=FileHandler(filename='my_log')
F.emit(onelog)
class LogHandler(logging.Handler):
"""multiprocessing log handler
This handler makes it possible for several processes
to log to the same file by using a queue.
"""
def __init__(self, fname):
logging.Handler.__init__(self)
self._handler = FH(self._make_filename(fname))
self.queue = multiprocessing.Queue(-1)
thrd = threading.Thread(target=self.receive)
thrd.daemon = True
thrd.start()
@classmethod
def _make_filename(self, fname):
filename = []
filename.append(fname)
filename.append('_')
unix_timestamp = int(time.time())
filename.append(str(unix_timestamp))
filename.append('.log')
return ''.join(filename)
def setFormatter(self, fmt):
logging.Handler.setFormatter(self, fmt)
self._handler.setFormatter(fmt)
def receive(self):
while True:
try:
record = self.queue.get()
self._handler.emit(record)
except (KeyboardInterrupt, SystemExit):
raise
except EOFError:
break
except Exception:
traceback.print_exc(file=sys.stderr)
def send(self, s):
self.queue.put_nowait(s)
def _format_record(self, record):
if record.args:
record.msg = record.msg % record.args
record.args = None
if record.exc_info:
self.format(record)
record.exc_info = None
return record
def emit(self, record):
try:
s = self._format_record(record)
self.send(s)
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
self.handleError(record)
def close(self):
self._handler.close()
logging.Handler.close(self)
