def config_tornado_log(options):
# 配置tornado日志格式，使用TimedRotating的方式来按天切割日志
import logging
from tornado.log import LogFormatter
if options is None:
from tornado.options import options
if options.logging is None or options.logging.lower() == 'none':
return
logger = logging.getLogger()
logger.setLevel(getattr(logging, options.logging.upper()))
if options.log_file_prefix:
print "Set logging config with file at %s" % options.log_file_prefix
channel = logging.handlers.TimedRotatingFileHandler(
filename=options.log_file_prefix,
when='midnight',
interval=1,
backupCount=10)
channel.setFormatter(LogFormatter(color=False))
logger.addHandler(channel)
if (options.log_to_stderr or
(options.log_to_stderr is None and not logger.handlers)):
channel = logging.StreamHandler()
channel.setFormatter(LogFormatter())
logger.addHandler(channel)
logging.info("Set logging config with stdout.")
def enable_pretty_logging(options=None, logger=None):
if options is None:
from tornado.options import options
if options.logging is None or options.logging.lower() == 'none':
return
if logger is None:
logger = logging.getLogger()
logger.setLevel(getattr(logging, options.logging.upper()))
if options.log_file_prefix:
rotate_mode = options.log_rotate_mode
if rotate_mode == 'size':
channel = logging.handlers.RotatingFileHandler(
filename=options.log_file_prefix,
maxBytes=options.log_file_max_size,
backupCount=options.log_file_num_backups)
elif rotate_mode == 'time':
channel = logging.handlers.TimedRotatingFileHandler(
filename=options.log_file_prefix,
when=options.log_rotate_when,
interval=options.log_rotate_interval,
backupCount=options.log_file_num_backups)
else:
error_message = 'The value of log_rotate_mode option should be ' + \
'"size" or "time", not "%s".' % rotate_mode
raise ValueError(error_message)
channel.setFormatter(LogFormatter(color=False))
logger.addHandler(channel)
if (options.log_to_stderr
or (options.log_to_stderr is None and not logger.handlers)):
# Set up color if we are in a tty and curses is installed
channel = logging.StreamHandler()
channel.setFormatter(LogFormatter())
logger.addHandler(channel)
def patch_enable_pretty_logging(logger=None, fmt=None):
if logger is None:
return
if not logger.handlers:
channel = logging.StreamHandler()
if fmt:
channel.setFormatter(LogFormatter(fmt=fmt))
else:
channel.setFormatter(LogFormatter())
logger.addHandler(channel)
class Logs(object):
if not os.path.exists(env.log_dir):
os.makedirs(env.log_dir)
format = '%(asctime)-15s %(levelname)s <%(module)s.py - %(funcName)s> '\
'%(processName)s %(message)s'
# prepare logs for tornado general
formatter = LogFormatter(fmt=format, color=True)
general_logger = logging.getLogger('tornado.general')
general_logger.setLevel(env.general_level)
general_logger.propagate = False
fpath = os.path.join(env.log_dir, 'general.log')
general_handler = logging.handlers.RotatingFileHandler(fpath,
'a',
env.log_file_size,
env.log_count)
general_handler.setFormatter(formatter)
general_logger.addHandler(general_handler)
# prepare logs for tornado access
access_formatter = LogFormatter(fmt=format, color=True)
access_logger = logging.getLogger('tornado.access')
access_logger.setLevel(env.access_level)
access_logger.propagate = False
fpath = os.path.join(env.log_dir, 'access.log')
access_handler = logging.handlers.RotatingFileHandler(fpath,
'a',
env.log_file_size,
env.log_count)
access_handler.setFormatter(access_formatter)
access_logger.addHandler(access_handler)
# prepare logs for tornado application
application_formatter = LogFormatter(fmt=format, color=True)
application_logger = logging.getLogger('tornado.application')
application_logger.setLevel(env.application_level)
application_logger.propagate = False
fpath = os.path.join(env.log_dir, 'application.log')
application_handler = logging.handlers.RotatingFileHandler(fpath,
'a',
env.log_file_size,
env.log_count)
application_handler.setFormatter(application_formatter)
application_logger.addHandler(application_handler)
def config_logger(namespace=NETLOGGER_NAMESPACE, level=None, filename=None):
tmpLog = nllog.get_logger(namespace)
tmpLog.propagate = False
nllog.PROJECT_NAMESPACE = namespace
if filename:
add_filehandler(tmpLog, filename)
else:
handler = logging.StreamHandler()
handler.setFormatter(LogFormatter("%(message)s"))
tmpLog.addHandler(handler)
if level == "WARN":
tmpLog.setLevel(logging.WARN)
elif level == "TRACE":
tmpLog.setLevel(nllog.TRACE)
elif level == "DEBUG":
tmpLog.setLevel(logging.DEBUG)
if not filename:
enable_pretty_logging()
elif level == "CONSOLE":
tmpLog.setLevel(25)
else:
tmpLog.setLevel(logging.INFO)
return tmpLog
def init_log(self):
datefmt = '%Y-%m-%d %H:%M:%S'
fmt = '%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s'
formatter = LogFormatter(color=True, datefmt=datefmt, fmt=fmt)
root_log = logging.getLogger()
for logHandler in root_log.handlers:
logHandler.setFormatter(formatter)
def setup_syslog():
logger = getLogger('tornado.access')
logger.setLevel(INFO)
handler = SysLogHandler('/dev/log')
handler.setFormatter(
LogFormatter(color=False, fmt='{} %(message)s'.format(__name__)))
logger.addHandler(handler)
def init_log():
# init log
from tornado.log import LogFormatter
logger = logging.getLogger()
logger.setLevel('INFO')
# out log file
path = os.path.join(os.path.dirname(__file__), "log",
'admin_%s.log' % options.port)
channel = logging.handlers.TimedRotatingFileHandler(
filename=path,
when='midnight',
interval=1,
backupCount=options.log_file_num_backups)
channel.setFormatter(LogFormatter(color=False))
# out stderr
rf_handler = logging.StreamHandler(sys.stderr)
rf_handler.setLevel(logging.DEBUG)
rf_handler.setFormatter(
logging.Formatter("%(asctime)s - %(name)s - %(message)s"))
logger.addHandler(rf_handler)
logger.addHandler(channel)
def serve(self, port: int = 5000, n_process: int = None):
if n_process is None:
n_process = 1 if sys.platform == 'win32' else 8
fmt = LogFormatter(fmt='%(asctime)s - %(levelname)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
color=True)
root_logger = logging.getLogger()
console_handler = logging.StreamHandler()
file_handler = logging.FileHandler('server.log')
console_handler.setFormatter(fmt)
file_handler.setFormatter(fmt)
root_logger.addHandler(console_handler)
root_logger.addHandler(file_handler)
app_log.setLevel(logging.INFO)
gen_log.setLevel(logging.INFO)
access_log.setLevel(logging.INFO)
app_log.info("Model is loading...")
app_log.info("Model Has Been Loaded!")
app = Application([(r"/.*", LTPHandler, dict(ltp=self))])
server = HTTPServer(app)
server.bind(port)
server.start(n_process)
ioloop.IOLoop.instance().start()
def parse_logger_callback(self):
if options.disable_log:
options.logging = None
if options.log_file_prefix and options.log_port_prefix:
options.log_file_prefix += ".%s" % options.port
if options.log_patch:
logging.handlers.TimedRotatingFileHandler = ProcessLogTimedFileHandler
tornado_logger = logging.getLogger('tornado')
enable_pretty_logging(logger=tornado_logger)
logdir = options.logging_dir or settings.LOGGING_DIR
for log in settings.LOGGING:
opt = OptionParser()
define_logging_options(opt)
self.define(opt)
opt.log_rotate_when = log.get('when', 'midnight')
opt.log_to_stderr = log.get('log_to_stderr', False)
opt.logging = log.get('level', 'INFO')
opt.log_file_prefix = os.path.join(logdir, log['filename'])
if log.get('backups'):
opt.log_file_num_backups = log.get('backups')
if opt.log_port_prefix:
opt.log_file_prefix += ".%s" % options.port
opt.log_rotate_interval = log.get('interval', 1)
opt.log_rotate_mode = 'time'
logger = logging.getLogger(log['name'])
if not settings.DEBUG:
logger.propagate = 0
else:
if not opt.log_to_stderr:
logger.propagate = 0
else:
logger.propagate = 1
enable_pretty_logging(options=opt, logger=logger)
map(lambda h: h.setFormatter(LogFormatter(fmt=log.get("formatter", LogFormatter.DEFAULT_FORMAT),
color=settings.DEBUG)), logger.handlers)
def init_logger(cls, port):
formatter = LogFormatter(fmt=cls.__fmt.format(port=port),
datefmt="",
color=False)
access_log_handler = ConcurrentRotatingFileHandler(
filename=os.path.join(ServerConfig["log_dir"], "access.log"))
access_log_handler.setFormatter(formatter)
access_log.addHandler(access_log_handler)
server_log_handler = ConcurrentRotatingFileHandler(
filename=os.path.join(ServerConfig['log_dir'], 'server.log'),
maxBytes=128 * 1024 * 1024,
backupCount=5,
encoding='utf8')
server_log_handler.setFormatter(formatter)
gen_log.addHandler(server_log_handler)
app_log.addHandler(server_log_handler)
access_log.setLevel(logging.INFO)
gen_log.setLevel(getattr(logging, ServerConfig['log_level'].upper()))
app_log.setLevel(getattr(logging, ServerConfig['log_level'].upper()))
access_log.propagate = app_log.propagate = gen_log.propagate = False
return
def _parse_logger_callback(self):
# print('parse_logger_callback: ')
if options.disable_log:
options.logging = None
if options.log_file_prefix and options.log_port_prefix:
options.log_file_prefix += ".%s" % options.port
tornado_logger = logging.getLogger('tornado')
enable_pretty_logging(logger=tornado_logger)
logdir = options.logging_dir or settings.LOGGING_DIR
for log in settings.log_cfg.logging:
opt = OptionParser()
define_logging_options(opt)
self._define(opt)
opt.log_rotate_when = log.get('when', 'midnight')
opt.log_to_stderr = log.get(
'log_to_stderr', False
) if options.log_to_stderr is None else options.log_to_stderr
opt.logging = log.get('level', 'INFO')
opt.log_file_prefix = os.path.join(logdir, log['filename'])
if log.get('backups'):
opt.log_file_num_backups = log.get('backups')
if opt.log_port_prefix:
opt.log_file_prefix += ".%s" % options.port
opt.log_rotate_interval = log.get('interval', 1)
opt.log_rotate_mode = 'time'
logger = logging.getLogger(log['name'])
logger.propagate = 0
enable_pretty_logging(options=opt, logger=logger)
map(
lambda h: h.setFormatter(
LogFormatter(fmt=log.get("formatter", settings.
standard_format),
color=settings.DEBUG)), logger.handlers)
def main():
logg = logging.getLogger("signalqueue")
# Set up color if we are in a tty and curses is installed
color = False
if curses and sys.stderr.isatty():
try:
curses.setupterm()
if curses.tigetnum("colors") > 0:
color = True
except:
pass
channel = logging.StreamHandler()
channel.setFormatter(LogFormatter(color=color))
logg.addHandler(channel)
logg.info("YO DOGG.")
from django.conf import settings
try:
tornado.options.parse_command_line()
http_server = HTTPServer(Application())
http_server.listen(settings.SQ_WORKER_PORT)
IOLoop.instance().start()
except KeyboardInterrupt:
print 'NOOOOOOOOOOOO DOGGGGG!!!'
def setUp(self):
self.formatter = LogFormatter(color=False)
# Fake color support.
# We can't guarantee anything about the $TERM
# variable when the tests are run, so just patch in some values
# for testing. (testing with color off fails to expose some potential
# encoding issues from the control characters)
self.formatter._colors = {
logging.ERROR: u("\u0001"),
}
self.formatter._normal = u("\u0002")
# construct a Logger directly to bypass getLogger's caching
# 定义一个logger，带上标识，标识唯一的logger对象
#self.logger = logging.Logger('LogFormatterTest')
self.logger = logging.Logger('test')
self.logger.propagate = False
self.tempdir = tempfile.mkdtemp()
print(self.tempdir)
# self.filename = os.path.join(self.tempdir, 'log.out')
self.filename = './log.out'
print(self.filename)
self.handler = self.make_handler(self.filename)
self.handler.setFormatter(self.formatter)
# 添加日志处理器
self.logger.addHandler(self.handler)
def enable_pretty_logging(options=None, logger=None):
"""改进了tornado的日志等级和格式
"""
if options is None:
import tornado.options
options = tornado.options.options
if options.logging is None or options.logging.lower() == 'none':
return
if logger is None:
logger = logging.getLogger()
logger.setLevel(getattr(logging, options.logging.upper()))
if options.log_file_prefix:
rotate_mode = options.log_rotate_mode
if rotate_mode == 'size':
channel = logging.handlers.RotatingFileHandler(
filename=options.log_file_prefix,
maxBytes=options.log_file_max_size,
backupCount=options.log_file_num_backups)
elif rotate_mode == 'time':
channel = logging.handlers.TimedRotatingFileHandler(
filename=options.log_file_prefix,
when=options.log_rotate_when,
interval=options.log_rotate_interval,
backupCount=options.log_file_num_backups)
else:
error_message = 'The value of log_rotate_mode option should be ' +\
'"size" or "time", not "%s".' % rotate_mode
raise ValueError(error_message)
channel.setFormatter(LogFormatter(LOGFORMAT, color=False))
channel.setLevel(getattr(logging, options.logging.upper()))
logger.addHandler(channel)
def enable_pretty_logging_path(options=None, logger=None):
"""Turns on formatted logging output as configured.
This is called automaticaly by `tornado.options.parse_command_line`
and `tornado.options.parse_config_file`.
"""
if options is None:
from tornado.options import options
if options.logging == 'none':
return
if logger is None:
logger = logging.getLogger()
logger.setLevel(getattr(logging, options.logging.upper()))
if options.log_file_prefix:
channel = None
rotating_handler = config['rotating_handler']
if hasattr(logging.handlers, rotating_handler):
handler = getattr(logging.handlers, rotating_handler)
if rotating_handler == 'RotatingFileHandler':
channel = handler(
filename=options.log_file_prefix,
maxBytes=options.log_file_max_size,
backupCount=options.log_file_num_backups,
delay=config['delay'])
elif rotating_handler == 'TimedRotatingFileHandler':
channel = handler(
filename=options.log_file_prefix,
when=config['when'],
interval=config['interval'],
delay=config['delay'],
backupCount=options.log_file_num_backups)
channel.suffix = config['suffix']
if channel:
channel.setFormatter(LogFormatter(color=False))
logger.addHandler(channel)
if (options.log_to_stderr or
(options.log_to_stderr is None and not logger.handlers)):
# Set up color if we are in a tty and curses is installed
channel = logging.StreamHandler()
channel.setFormatter(LogFormatter())
logger.addHandler(channel)
def __init__(self, filename=None, name='root', level='INFO', stream=True):
level = getattr(logging, level.upper())
super(Logger, self).__init__(name, level)
datefmt = '%Y-%m-%d %H:%M:%S'
fmt = '%(color)s[%(levelname)s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s'
if stream:
hdlr = logging.StreamHandler()
hdlr.setFormatter(LogFormatter(fmt=fmt, datefmt=datefmt))
self.addHandler(hdlr)
if filename:
hdlr = logging.handlers.WatchedFileHandler(filename=filename,
mode='a',
encoding='utf-8')
hdlr.setFormatter(
LogFormatter(fmt=fmt, datefmt=datefmt, color=False))
self.addHandler(hdlr)
def init_logging():
tornado.options.parse_command_line()
from tornado.log import LogFormatter
access_log = logging.getLogger()
datefmt = '[%Y-%m-%d %H:%M:%S %z]'
fmt = '%(asctime)s [%(levelname)s %(name)s pid:%(process)d port:{port} %(filename)s:%(lineno)d] %(message)s'.format(
port=options.port or '')
formatter = LogFormatter(color=True, fmt=fmt, datefmt=datefmt)
access_log.handlers[0].setFormatter(formatter)
def _setup_logging(self):
logger.setLevel(logging.INFO)
channel = logging.StreamHandler()
channel.setFormatter(LogFormatter())
logger.addHandler(channel)
# need a tornado logging handler to prevent IOLoop._setup_logging
logging.getLogger('tornado').addHandler(channel)
def _set_log():
enable_pretty_logging()
fmt = LogFormatter()
for logger_name in ['tornado.access', 'tornado.application', 'tornado.general']:
logger = logging.getLogger(logger_name)
log_file_path = os.path.join(BASE_LOG_PATH, logger_name + '.log')
log_handler = logging.FileHandler(log_file_path)
log_handler.setFormatter(fmt)
logger.addHandler(log_handler)
def __init__(self):
GENERAL_LOG = settings.LOGGER_MODULE['GENERAL_LOG']
log_file = GENERAL_LOG['FILE']
log_name = GENERAL_LOG['NAME']
rollover_when = GENERAL_LOG['ROLLOVER_WHEN']
log_formatter = LogFormatter(fmt=DEFAULT_FORMAT,
datefmt=DEFAULT_DATE_FORMAT,
color=settings.TORNADO_CONF['debug'])
self.init(log_file, log_formatter, log_name, rollover_when)
def set_log():
if options.log_path:
nor = logging.getLogger('tornado.access')
nor.setLevel(logging.INFO)
filehandler = logging.handlers.TimedRotatingFileHandler(
options.log_path, 'midnight', 1, 0)
filehandler.suffix = "%Y%m%d.log"
filehandler.setFormatter(LogFormatter())
nor.addHandler(filehandler)
def init_log():
ch = logging.StreamHandler()
formatter = LogFormatter(
fmt=
'%(color)s[%(levelname)1.1s %(asctime)s StochSS %(filename)s:%(lineno)d]%(end_color)s %(message)s',
datefmt='%H:%M:%S')
ch.setFormatter(formatter)
log.setLevel(logging.WARNING)
log.addHandler(ch)
log.propagate = False
def enable_pretty_logging(options=options):
"""Turns on formatted logging output as configured.
This is called automatically by `parse_command_line`.
"""
root_logger = logging.getLogger()
if options.log_file_prefix:
channel = logging.handlers.RotatingFileHandler(
filename=options.log_file_prefix,
maxBytes=options.log_file_max_size,
backupCount=options.log_file_num_backups)
channel.setFormatter(LogFormatter(color=False))
root_logger.addHandler(channel)
if (options.log_to_stderr
or (options.log_to_stderr is None and not root_logger.handlers)):
# Set up color if we are in a tty and curses is installed
channel = logging.StreamHandler()
channel.setFormatter(LogFormatter())
root_logger.addHandler(channel)
def _setup_logging():
root_logger = logging.getLogger()
root_logger.setLevel(logging.NOTSET)
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
handler.setFormatter(
LogFormatter(
fmt='%(color)s[%(asctime)s %(name)s]%(end_color)s %(message)s',
datefmt='%H:%M:%S'))
root_logger.addHandler(handler)
def setup_logging(debug=False):
"""Configure logging handy for micro apps.
By default, all :attr:`logging.INFO` messages are logged, along with only :attr:`loggin.ERROR`
messages for the access log. In *debug* mode, the access log is not filtered.
"""
logger = getLogger()
handler = StreamHandler()
handler.setFormatter(LogFormatter())
logger.addHandler(handler)
logger.setLevel(logging.INFO)
if not debug:
getLogger('tornado.access').setLevel(logging.ERROR)
def initialize(self):
# initialize the logger with tornado's LogFormatter
self.formatter = LogFormatter(color=False)
self.logger = logging.Logger(self.__class__.__name__)
self.logger.propagate = False
try:
self.handler = logging.FileHandler("/var/log/yongwu_" + date.today().isoformat() + ".log")
except IOError:
print "Please run as administrator."
self.handler.setFormatter(self.formatter)
self.logger.addHandler(self.handler)
def start_server():
options = init_tornado_options()
# init socket options
socket.setdefaulttimeout(options.socket_timeout)
# server settings for application
app_settings = {
'debug': options.debug,
'template_path': os.path.join(os.path.dirname(__file__), 'template'),
'static_path': os.path.join(os.path.dirname(__file__), 'static'),
'cookie_secret': 'ngYrl3h4TRGF9KM6zb5x2Q/v5sH8T0BbsOisjQIL95Q='
}
# cookie_secret generated by "base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes)"
logger = logging.getLogger()
logger.setLevel(logging.INFO)
log_file = 'sys.log'
# timelog = logging.handlers.TimedRotatingFileHandler(log_file, 'midnight', 1, 0)
# logger.addHandler(timelog)
datefmt = '%Y-%m-%d %H:%M:%S'
fmt = '%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s'
formatter = LogFormatter(color=True, datefmt=datefmt, fmt=fmt)
root_log = logging.getLogger()
for logHandler in root_log.handlers:
logHandler.setFormatter(formatter)
logging.info('options: {}'.format(options.items()))
# set default handler, just response 404
router = list()
# load routing
load_routings(router=router)
# else
router.append((ur'/.*', handlers.APINotFoundHandler))
# setting to handlers
handlers.APIBaseHandler.static_config.update(config.config)
def shutdown(sig, frame):
logging.info("shutting down on signal:%s", sig)
tornado.ioloop.IOLoop.instance().stop()
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
# create tornado application and httpserver
application = tornado.web.Application(router, **app_settings)
httpserver = tornado.httpserver.HTTPServer(application)
httpserver.bind(options.port, backlog=options.backlog)
httpserver.start(1 if options.debug else options.workers)
# ioloop start
tornado.ioloop.IOLoop.instance().start()
def bootstrap_core_logging():
"""This is a replacement for standard Tornado logging configuration."""
handlers = []
level = getattr(logging, options.loglevel.upper())
context_filter = ContextFilter()
ROOT_LOGGER.setLevel(logging.NOTSET)
if options.logfile:
file_handler = logging.handlers.WatchedFileHandler(options.logfile)
file_handler.setFormatter(logging.Formatter(options.logformat))
handlers.append(file_handler)
if options.stderr_log:
stderr_handler = logging.StreamHandler()
stderr_handler.setFormatter(
LogFormatter(fmt=options.stderr_format,
datefmt=options.stderr_dateformat))
handlers.append(stderr_handler)
if options.syslog:
if options.syslog_port is not None:
syslog_address = (options.syslog_address, options.syslog_port)
else:
syslog_address = options.syslog_address
try:
syslog_formatter = logging.Formatter('{}: {}'.format(
options.app, options.logformat))
syslog_handler = SysLogHandler(
facility=SysLogHandler.facility_names[options.syslog_facility],
address=syslog_address)
syslog_handler.setFormatter(syslog_formatter)
handlers.append(syslog_handler)
except socket.error:
logging.getLogger('frontik.logging').exception(
'cannot initialize syslog')
for logger_name in options.suppressed_loggers:
logging.getLogger(logger_name).setLevel(logging.WARN)
for handler in handlers:
handler.setLevel(level)
handler.addFilter(context_filter)
ROOT_LOGGER.addHandler(handler)
ROOT_LOGGER.addHandler(GlobalLogHandler())
if not ROOT_LOGGER.handlers:
ROOT_LOGGER.addHandler(logging.NullHandler())
def capture_log(app, fmt="[%(levelname)s] %(message)s"):
"""Adds an extra handler to the given application the logs to a string
buffer, calls ``app.start()``, and returns the log output. The extra
handler is removed from the application before returning.
Arguments
---------
app: LoggingConfigurable
An application, withh the `.start()` method implemented
fmt: string
A format string for formatting log messages
Returns
-------
A dictionary with the following keys (error and log may or may not be present):
- success (bool): whether or not the operation completed successfully
- error (string): formatted traceback
- log (string): captured log output
"""
log_buff = six.StringIO()
handler = logging.StreamHandler(log_buff)
formatter = LogFormatter(fmt="[%(levelname)s] %(message)s")
handler.setFormatter(formatter)
app.log.addHandler(handler)
try:
app.start()
except:
log_buff.flush()
val = log_buff.getvalue()
result = {"success": False}
result["error"] = traceback.format_exc()
if val:
result["log"] = val
else:
log_buff.flush()
val = log_buff.getvalue()
result = {"success": True}
if val:
result["log"] = val
finally:
log_buff.close()
app.log.removeHandler(handler)
return result
