def init_logger():
# 创建 scrapy 日志记录器
logger = logging.getLogger('scrapy')
logger.setLevel(logging.INFO)
# 设置日志的格式化对象
# 要求收集日志的时间、等级名称、发生所在的函数名及行号
fmt_str = '%(asctime)s %(levelname)s %(funcName)s %(lineno)s: %(message)s'
formatter = Formatter(fmt=fmt_str, datefmt='%Y-%m-%d %H:%M:%S')
handler1 = StreamHandler()
handler1.setLevel(logging.INFO)
handler1.setFormatter(formatter)
handler2 = FileHandler('errors.log', encoding='utf-8')
handler2.setLevel(logging.ERROR)
handler2.setFormatter(formatter)
handler3 = HTTPHandler(host='10.12.152.218:5000',
url='/log/',
method='POST')
# handler3.setFormatter(formatter)
handler3.setLevel(logging.ERROR)
# 将处理器添加到日志记录器中
logger.addHandler(handler1)
logger.addHandler(handler2)
logger.addHandler(handler3)
def __init__(self, db_host, db_port, db_name, collection, host, url, method):
HTTPHandler.__init__(self, host, url, method)
self.db_host = db_host
self.db_port = db_port
self.db_name = db_name
self.collection = collection
self.s = requests.Session()
def __init__(self, url, username=None, icon_url=None, icon_emoji=None, channel=None, mention=None):
o = urlparse(url)
is_secure = o.scheme == 'https'
HTTPHandler.__init__(self, o.netloc, o.path, method="POST", secure=is_secure)
self.username = username
self.icon_url = icon_url
self.icon_emoji = icon_emoji
self.channel = channel
self.mention = mention and mention.lstrip('@')
def test4(self):
# formatter、handler、logger 之间关系?
# handler 对象可以添加formatter
# logger 可以添加多个 handler, 如果没有handler,默认会添加 StreamHandler
# 常用的Handler有哪些
# 1) StreamHandler -> 控制台打印输出的处理器
# 2) FileHandler -> 日志文件输出的处理器
# 3）HttpHandler -> 网络上传日志的处理器
logger = logging.getLogger('disen')
# user_id 是自定义格式变量
log_format_str = '< %(user_id)s >[ %(asctime)s ]-%(levelname)s: %(message)s'
log_format_date = '%Y-%m-%d %H:%M:%S'
# 实例化handler - 标准输出的
handler1 = logging.StreamHandler()
# 设置handler的日志等级、格式
handler1.setLevel(logging.INFO)
handler1.setFormatter(
logging.Formatter(fmt=log_format_str, datefmt=log_format_date))
logger.addHandler(handler1)
handler2 = logging.FileHandler('handler.log')
handler2.setFormatter(
logging.Formatter(fmt=log_format_str, datefmt=log_format_date))
handler2.setLevel(logging.WARN) # 文件处理器只记录 警告信息
logger.addHandler(handler2)
# 创建上传日志的处理器
# 上传日志信息中的所有信息
httpHandler = HTTPHandler(host='10.12.155.80:5000',
url='/upload_log/',
method='POST')
httpHandler.setLevel(logging.ERROR)
logger.addHandler(httpHandler)
# 1. 检查每个消息的等级，再判断当前日志记录器是否处理
# 2. 在日志的记录器中 获取它的所有处理器handler
# 3. 根据消息的等级，分别由不同的处理器处理消息。
extra_info = {'user_id': '1000001'}
logger.info('hi, info', extra=extra_info)
logger.warning('hi, warning', extra=extra_info)
logger.error('hi, error', extra=extra_info)
logger.critical('hi, critical', extra=extra_info)
def __init__(self, logPath, host, url, method, protocol='http'):
"""
Constructor
:param logPath: log path on HTTP server
:param host: Host name or IP
:param url: URL of web services
:param method: HTTP method
:param protocol: HTTP or HTTPS
"""
HTTPHandler.__init__(self, host, url, method)
self.logPath = logPath
self.session = requests.Session()
self.protocol = protocol
def mapLogRecord(self, record):
record_modified = HTTPHandler.mapLogRecord(self, record)
record_modified['logPath'] = self.logPath
try:
record_modified['msg'] = self.format(record)
except:
pass
return record_modified
def mapLogRecord(self, record):
record_modified = HTTPHandler.mapLogRecord(self, record)
record_modified['logPath'] = self.logPath
try:
record_modified['msg'] = self.format(record)
except:
pass
return record_modified
def mapLogRecord(self, record):
record_modified = HTTPHandler.mapLogRecord(self, record)
try:
record_modified['msg'] = self.format(record)
except:
pass
record_modified['exc_info'] = format_exc_info(record_modified['exc_info'])
record_modified['args'] = str(record_modified['args'])
return record_modified
def machine_thread():
logger_get.addHandler(HTTPHandler("localhost:8888", "/", method='POST'))
try:
server = SocketServer.TCPServer(("", PORT), myHandler)
print '\nStarted MACHINE - httpserver on port ', PORT
#Wait forever for incoming http requests
server.serve_forever()
except KeyboardInterrupt:
print '^C received, shutting down the web server'
server.socket.close()
def mapLogRecord(self, record):
"""Map log record as required format of HTTP/HTTPS server"""
record_modified = HTTPHandler.mapLogRecord(self, record)
if self.formatter is not None:
record_modified['t'] = self.formatter.format(record)
else:
record_modified['t'] = record_modified['msg'].encode('utf-8')
record_modified['no_date'] = self.no_date
return record_modified
def mapLogRecord(self, record):
"""
Map log record as required format of HTTP/HTTPS server
:param record:
:return:
"""
record_modified = HTTPHandler.mapLogRecord(self, record)
record_modified['logPath'] = self.logPath
record_modified['msg'] = record_modified['msg'].encode('utf-8')
return record_modified
def send_events_http():
global results
start_time = time.time()
logger = logging.getLogger('video_analytics')
# http_handler = HTTPHandler('', '', method='GET')
# logging.handlers.HTTPHandler(http_handler)
http_handler = HTTPHandler(HTTP_ADDR, '/httpevent', method='POST')
logger.addHandler(http_handler)
#logger.setLevel(logging.DEBUG)
# test = ["actor=bob", "action=submitanswer", "object=problem5"]
# objects = [o.split("=") for o in test]
# logger.error(json.dumps(dict(objects)))
print "================================"
print len(results), "incoming entries"
# print results
for entry in results:
logger.critical(json.dumps(entry))
print sys._getframe().f_code.co_name, "COMPLETED", (time.time() - start_time), "seconds"
def __create_logger(name):
env = EnvSetUp.Instance()
config_spec = env.get_config_spec()
logserver_ip = config_spec["LOGGING_CONFIGURATION"][
"LOGSERVER_CONFIGURATION"]["SERVER_IP"]
logserver_port = config_spec["LOGGING_CONFIGURATION"][
"LOGSERVER_CONFIGURATION"]["SERVER_PORT"]
logserver_uri = config_spec["LOGGING_CONFIGURATION"][
"LOGSERVER_CONFIGURATION"]["URI_ENDPOINT"]
log_level = config_spec["LOGGING_CONFIGURATION"]["LOG_LEVEL"]
logger = logging.getLogger(name)
logger.setLevel(log_level)
if logger.handlers == []:
server = logserver_ip + ":" + str(logserver_port)
print server
http_handler = HTTPHandler(host=server,
url=logserver_uri,
method="POST")
logger.addHandler(http_handler)
return logger
def config_log():
fmt = Formatter(fmt='%(asctime)s %(name)s %(levelname)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
io_handler = StreamHandler()
io_handler.setLevel(logging.DEBUG)
io_handler.setFormatter(fmt)
file_handler = FileHandler('edu_logs.log')
file_handler.setLevel(logging.WARN)
file_handler.setFormatter(fmt)
http_handler = HTTPHandler(host='localhost:5000',
url='/log',
method='POST')
http_handler.setLevel(logging.ERROR)
http_handler.setFormatter(fmt) # 上传的数据不需要formatter
logger.setLevel(logging.DEBUG)
logger.addHandler(io_handler)
logger.addHandler(file_handler)
logger.addHandler(http_handler)
def test_log(self):
logger = logging.getLogger()
fileHandler = TimedRotatingFileHandler('logs/content.log')
fileHandler.setLevel(
logging.INFO
) # NOTSET -> DEBUG -> INFO -> WARNING -> ERROR -> FATAL
formatter = Formatter('%(asctime)s %(levelname)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
fileHandler.setFormatter(formatter)
iohandler = StreamHandler()
iohandler.setLevel(logging.DEBUG)
logger.addHandler(fileHandler)
logger.addHandler(iohandler)
# 将日志上传到日志服务器
# host 服务器的ip和port
# url 请求路径，以 / 开始
# method 默认 为GET
httpHandler = HTTPHandler(host='127.0.0.1:5000',
url='/upload_log',
method='POST')
httpHandler.setFormatter(formatter)
# 只有error 的信息才会上传到日志服务器
httpHandler.setLevel(logging.ERROR)
logger.addHandler(httpHandler)
logger.setLevel(logging.DEBUG)
logger.info('this is a test log python file!')
logger.error('this is error msg for python file!')
logger.debug('this is dubug msg !') # this msg is hiddern!
handler3 = SMTPHandler('smtp.qq.com',
fromaddr='*****@*****.**',
toaddrs='*****@*****.**',
subject='日志邮箱通知')
# 配置邮件验证用户和口令（或授权码）
handler3.username = '******'
handler3.password = '******' # QQ邮箱发件时验证的“授权码”
handler3.setLevel(logging.FATAL)
# 创建上传日志请求的处理器HTTPHandler
# 上传日志的接口： http://10.35.162.97:5000/log/
# 接口的方法： post
# 响应的数据是json
handler4 = HTTPHandler(host='10.35.162.97:5000',
url='/log/', method='POST')
handler4.setLevel(logging.INFO)
# 创建格式化对象 , 并添加到处理器
formatter = logging.Formatter('[ <%(asctime)s> %(name)s %(levelname)s ] %(message)s')
handler1.setFormatter(formatter)
handler2.setFormatter(formatter)
handler3.setFormatter(formatter)
handler4.setFormatter(formatter) # 无效的，默认情况下将所有的日志信息上传到服务器
# 将处理器添加到日志记录器
logger.addHandler(handler1)
logger.addHandler(handler2)
# logger.addHandler(handler3)
logger.addHandler(handler4)
# -*- coding: utf-8 -*-
from logging.handlers import HTTPHandler
import requests
import logging
#1.获取或创建xx记录器
logger = logging.getLogger("request")
#1.1设置记录器的日志等级（FATAL 50、ERROR 40、WARN 30、INFO 20、DEBUG 10、NOSET 0 ）
#等级高的，不会获取等级低的日志
logger.setLevel(logging.INFO)
#2.设置日志的格式
fmt_str ='[%(asctime)s%(levelname)s]%(message)s'
format = logging.Formatter(fmt = fmt_str,datefmt="%Y-%m-%d %H:%M:%S")
#3.设置日志处理器Handler
# - StreamHandler 控制台输出流
# - FileHandler 日志文件输出
# - HttpHandler 网络请求的日志上传处理器
handler = logging.StreamHandler()
file_handler = logging.FileHandler("test.log")
file_handler.setLevel((logging.WARN)#设置处理日志等级
http_handler = HTTPHandler(host = )
format=logging.Formatter('')
def __init__(self, filename, host, url, method):
HTTPHandler.__init__(self, host, url, method)
self.logPath = filename
self.session = requests.Session()
# 1.1 设置记录器的日志等级（FATAL 50、ERROR 40、WARN 30, INFO 20, DEBUG 10, NOSET 0）
# 注： 等级高的， 不会获取等级低的日志
logger.setLevel(logging.INFO)
# 2. 设置日志的格式
fmt_str = '[ %(asctime)s %(levelname)s ] %(message)s'
format = logging.Formatter(fmt=fmt_str, datefmt='%Y-%m-%d %H:%M:%S')
# 3. 设置日志的处理器 Handler
# - StreamHandler 控制台输出流
# - FileHandler 日志文件输出
# - HttpHandler 网络请求的日志上传处理器
handler = logging.StreamHandler()
file_handler = TimedRotatingFileHandler('test.log') # 按时间分隔的日志文件
file_handler.setLevel(logging.WARN) # 设置处理的日志等级
http_handler = HTTPHandler(host='10.35.166.35:5000',
url='/upload_log/',
method='POST')
http_handler.setLevel(logging.ERROR)
logger.addHandler(handler)
logger.addHandler(file_handler)
logger.addHandler(http_handler)
# 程序中发出不同等级的日志的消息
logger.info('第一个普通消息的测试')
logger.debug('日志等级低的消息不会被处理')
logger.warning('警告日志会被写入文件中，同时控制台显示 ')
logger.critical('严重错误日志信息，被上传到日志服务器， 同时控制台显示')
"""
Author: LanHao
Date: 2020/11/16
Python: python3.6
"""
import time
import logging
from logging import Handler
from logging.handlers import HTTPHandler, QueueHandler
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
host = '127.0.0.1:8080'
url = '/logs/apis'
handler = HTTPHandler(host, url, method='POST')
logger.addHandler(handler)
for i in range(2):
logger.critical("你好")
# time.sleep(10)
def __init__(self, host, database):
HTTPHandler.__init__(self, host, database, 'POST')
from WMCore.Database.CMSCouch import CouchServer
self.database = CouchServer(dburl=host).connectDatabase(database, size=10)
from logging import FileHandler, StreamHandler
from logging.handlers import HTTPHandler, SMTPHandler
log = logging.Logger('Tornado', logging.DEBUG)
fmt = logging.Formatter(fmt='%(asctime)s-%(name)s-%(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
fileHandler = FileHandler('test.log')
fileHandler.setLevel(logging.WARNING)
fileHandler.setFormatter(fmt=fmt)
streamHandler = StreamHandler()
streamHandler.setLevel(logging.INFO)
streamHandler.setFormatter(fmt=fmt)
httpHandler = HTTPHandler(host='127.0.0.1:8000', url='/', method='POST')
httpHandler.setLevel(logging.ERROR)
if __name__ == '__main__':
log.addHandler(streamHandler)
log.addHandler(fileHandler)
log.addHandler(httpHandler)
log.info('hello info')
log.debug('hello debug')
log.warning('hello warning')
log.error('hello error')
log.critical('hello critical')
# Generates a dummy event for testing
# E.g.:
#
# python sendevent.py localhost:8020 /httpevent actor=bob action=submitanswer object=problem5
import json
import sys
import logging
from logging.handlers import HTTPHandler
logger = logging.getLogger('sendevent')
logger.addHandler(HTTPHandler(sys.argv[1], sys.argv[2]))
objects = [o.split("=") for o in sys.argv[3:]]
logger.error(json.dumps(dict(objects)))
def __init__(self, logPath, host, url, method):
HTTPHandler.__init__(self, host, url, method)
self.logPath = logPath
self.s = requests.Session()
def __init__(self, application, **kw):
self.application = application
# Error handling WSGI app
self._errapp = kw.get('errapp', _errapp)
# Flag controlling logging
self.log = kw.get('log', True)
# Log if set
if self.log:
# Log error message
self.message = kw.get('logmessage', ERRORMSG)
# Individual logger for WSGI app with custom name 'logname'
self.logger = logging.getLogger(kw.get('logname', LOGNAME))
# Set logger level
self.logger.setLevel(kw.get('loglevel', logging.DEBUG))
# Log formatter
format = logging.Formatter(
# Log entry format
kw.get('logformat', LOGFORMAT),
# Date format
kw.get('datefmt', DATEFORMAT))
# Coroutine for setting individual log handlers
def setlog(logger):
logger.setFormatter(format)
self.logger.addHandler(logger)
# Log to STDOUT
if 'tostream' in kw:
setlog(logging.StreamHandler())
# Log to a rotating file that with periodic backup deletions
if 'tofile' in kw:
setlog(
RotatingFileHandler(
# Log file path
kw.get('file', LOGNAME),
# Interval to backup log file
maxBytes=kw.get('maxBytes', MAXBYTES),
# Number of backups to keep
backupCount=kw.get('backupCount', BACKUPCOUNT)))
# Send log entries to an email address
if 'toemail' in kw:
setlog(
SMTPHandler(
# Mail server
kw.get('mailserver'),
# From email address
kw.get('frommail'),
# To email address
kw.get('toemail'),
# Email subject
kw.get('mailsubject')))
# Send log entries to a web server
if 'tohttp' in kw:
setlog(
HTTPHandler(
# Web server host
kw.get('httphost'),
# Web URL
kw.get('httpurl'),
# HTTP method
kw.get('httpmethod', 'GET')))
# Log to syslog
if 'tosyslog' in kw:
setlog(
SysLogHandler(
# syslog host
kw.get('syshost', ('localhost', 514)),
# syslog user
kw.get('facility', 'LOG_USER')))
assert self.logger.handlers, 'At least one logging handler must be configured'
# Redirect STDOUT to the logger
if 'toprint' in kw:
sys.stdout = LogStdout(
self.logger,
# Sets log level STDOUT is displayed under
kw.get('prnlevel', logging.DEBUG))
# Flag for turning on PDB in situ
self.debug = kw.get('debug', False)
# Flag for sending HTML-formatted exception tracebacks to the browser
self.tohtml = kw.get('tohtml', False)
# Write HTML-formatted exception tracebacks to a file if provided
self.htmlfile = kw.get('htmlfile')
def __init__(self, no_date='false', **kwargs):
HTTPHandler.__init__(self, **kwargs)
self.no_date = no_date
def __init__(self, log_path, host, url, method, day=None):
HTTPHandler.__init__(self, host, url, method)
self.logPath = log_path
self.day = day
self.s = requests.Session()
import logging
from logging.handlers import HTTPHandler
import sys
# logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
logger.setLevel(level=logging.DEBUG)
# StreamHandler
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setLevel(level=logging.DEBUG)
logger.addHandler(stream_handler)
# FileHandler
file_handler = logging.FileHandler('Output.log')
file_handler.setLevel(level=logging.INFO)
formatter = logging.Formatter(
fmt='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
datefmt='%Y/%m/%d %H:%M:%S')
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
# HTTPHandler
http_handler = HTTPHandler(host='localhost:8001', url='log', method='POST')
logger.addHandler(http_handler)
# Log
logger.info('This is a log info')
logger.debug('Debugging')
logger.warning('Warning exists')
logger.info('Finish')
prefix = fmt[:start]
suffix = fmt[end + 1:]
length = end - start + 1
# this is essentially the python format string equivalent to the
# template string
fmt_string = prefix + '{0:0' + str(length) + 'd}' + suffix
return fmt_string.format(num)
# setup the HTTP log handling
log = logging.getLogger()
log.setLevel(logging.DEBUG)
if HTTP_LOG_URL is not None:
log.addHandler(HTTPHandler(HTTP_LOG_URL, '/'))
# do the arg parsing by hand since neither getopt nor optparse support
# single dash long options.
args = sys.argv[1:]
if (len(args) % 2) != 0:
logging.error(
'the argument list is not well formed (odd number of args/options)')
sys.exit()
options = dict()
for x in range(0, len(args), 2):
options[args[x]] = args[x + 1]
def __init__(self, host, database):
HTTPHandler.__init__(self, host, database, 'POST')
from WMCore.Database.CMSCouch import CouchServer
self.database = CouchServer(dburl=host).connectDatabase(database,
size=10)
def addHttpLog(url):
http_handler = HTTPHandler(url, "/data/log", method='POST')
http_handler.setLevel(logging.ERROR)
logger.addHandler(http_handler)
import logging
from logging.handlers import HTTPHandler
import pymongo
# 自定义Handler
class MongoHandler(logging.Handler):
def emit(self, record):
pass
logger = logging.getLogger()
fh = logging.FileHandler('logger.log') # 文件输出流的对象
ch = logging.StreamHandler() # 标准输出流，用于输出到控制台
nw = HTTPHandler('localhost:8888', url='/') # 默认以GET方式输出到网页上
formatter = logging.Formatter(
fmt='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
datefmt='%Y%m%d %H:%M:%S')
logger.setLevel('DEBUG')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
nw.setFormatter(formatter)
logger.addHandler(fh) # 输出到文件
logger.addHandler(ch) # 输出到屏幕
logger.addHandler(nw) # GET带参数输出
logger.debug('debug message')