class ElasticSearchBackend(AbstractBackend):
"""
ElasticsearchBackend is used to create an abstraction of the handler and the formatter concepts from logging.
Here, we have a CMRESHandler which is part of an external library named 'cmreslogging' based on 'logging'.
CMRESHandler is a specific handler created to send log records to an ElasticSearch DB. It does not need a Formatter
object.
"""
def __init__(self):
"""
CMRESHandler needs, at least, a hostname, a username, a password, a port and a specific index
from the ElasticSearch DB to send log records.
"""
super(ElasticSearchBackend, self).__init__(None, None)
self.__host = ''
self.__user = None
self.__passwd = None
self.__port = 9203
self.__index = ''
self.__bufferSize = 1000
self.__flushTime = 1
def createHandler(self, parameters=None):
"""
Each backend can initialize its attributes and create its handler with them.
:params parameters: dictionary of parameters. ex: {'FileName': file.log}
"""
if parameters is not None:
self.__host = parameters.get('Host', self.__host)
self.__user = parameters.get('User', self.__user)
self.__passwd = parameters.get('Password', self.__passwd)
self.__port = int(parameters.get('Port', self.__port))
self.__index = parameters.get('Index', self.__index)
self.__bufferSize = int(
parameters.get('BufferSize', self.__bufferSize))
self.__flushTime = int(
parameters.get('FlushTime', self.__flushTime))
if self.__user is not None and self.__passwd is not None:
self._handler = CMRESHandler(
hosts=[{
'host': self.__host,
'port': self.__port
}],
auth_type=CMRESHandler.AuthType.BASIC_AUTH,
auth_details=(self.__user, self.__passwd),
es_index_name=self.__index,
use_ssl=True,
verify_ssl=True,
buffer_size=self.__bufferSize,
flush_frequency_in_sec=self.__flushTime)
else:
self._handler = CMRESHandler(
hosts=[{
'host': self.__host,
'port': self.__port
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=self.__index,
use_ssl=True,
verify_ssl=True,
buffer_size=self.__bufferSize,
flush_frequency_in_sec=self.__flushTime)
# We give a format containing only asctime to add the field in elasticsearch
# asctime is not created at the initialization of the LogRecords but built in the format process
self._handler.setFormatter(logging.Formatter('%(asctime)s'))
def setLevel(self, level):
"""
No possibility to set the level of the ElasticSearch handler.
It is not set by default so it can send all Log Records of all levels to ElasticSearch.
"""
pass
def setFormat(self, fmt, datefmt, options):
"""
Each backend give a format to their formatters and attach them to their handlers.
:params fmt: string representing the log format
:params datefmt: string representing the date format
:params component: string represented as "system/component"
:params options: dictionary of logging options. ex: {'Color': True}
"""
pass
def get_logger(name=None):
"""
get logger by name
:param name: name of logger
:return: logger
"""
# 开启只保留7日内Log,不需要可以注释掉
rmNonRecentLog()
global loggers
if not name: name = __name__
if loggers.get(name):
return loggers.get(name)
logger = logging.getLogger(name)
logger.setLevel(LOG_LEVEL)
# 输出到控制台
if LOG_ENABLED and LOG_TO_CONSOLE:
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setLevel(level=LOG_LEVEL)
formatter = logging.Formatter(LOG_FORMAT)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
# 输出到文件
if LOG_ENABLED and LOG_TO_FILE:
# 如果路径不存在，创建日志文件文件夹
log_dir = dirname(LOG_PATH)
if not exists(log_dir): makedirs(log_dir)
# 添加 FileHandler
file_handler = logging.FileHandler(LOG_PATH, encoding='utf-8')
file_handler.setLevel(level=LOG_LEVEL)
formatter = logging.Formatter(LOG_FORMAT)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
# 输出到 Elasticsearch
if LOG_ENABLED and LOG_TO_ES:
# 添加 CMRESHandler
es_handler = CMRESHandler(
hosts=[{
'host': ELASTIC_SEARCH_HOST,
'port': ELASTIC_SEARCH_PORT
}],
# 可以配置对应的认证权限
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=ELASTIC_SEARCH_INDEX,
# 一个月分一个 Index
index_name_frequency=CMRESHandler.IndexNameFrequency.MONTHLY,
# 额外增加环境标识
es_additional_fields={'environment': APP_ENVIRONMENT})
es_handler.setLevel(level=LOG_LEVEL)
formatter = logging.Formatter(LOG_FORMAT)
es_handler.setFormatter(formatter)
logger.addHandler(es_handler)
# 保存到全局 loggers
loggers[name] = logger
return logger
async def InitServer(cls,
servertype="unkown-servertype",
startmysql=True,
startpg=True):
# 创建日志目录
if not os.path.exists('logs'):
os.makedirs('logs')
logging.basicConfig(level=logging.DEBUG) # 默认屏幕输出
filehandler = TimedRotatingFileHandler( # 时间轮转输出
filename="logs/my.log",
when='D',
interval=1,
backupCount=0)
filehandler.suffix = "%Y%m%d-%H%M%S.log"
formatter = logging.Formatter(
"%(asctime)s-%(name)s-%(levelname)s-[%(filename)s:%(lineno)d]-%(message)s"
)
filehandler.setFormatter(formatter)
filehandler.setLevel(logging.INFO)
logging.getLogger().addHandler(filehandler) # 添加时间轮转输出
# 服务器名字 数据文件路径
cls.myservertype = servertype
cls.mydatapath = datafilepath()
# 生成一个新的 guid 可能不会使用，如果有的话
newguid = cls.myservertype + "_" + str(uuid.uuid1())
await cls.loadconfig()
if (cls.elkdatabase == ""):
cls.elkdatabase = cls.myservertype
if (cls.myserverguid == None):
cls.myserverguid = newguid
# 没有配置文件就保存
await cls.saveconfig()
# ELK的日志输出
es_handler = CMRESHandler(hosts=[{
'host': cls.elkhost,
'port': cls.elkport
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=cls.elkdatabase,
es_additional_fields={
'App': cls.myservertype,
'AppGuid': cls.myserverguid
})
es_handler.setLevel(logging.INFO)
formatter2 = logging.Formatter(
"%(asctime)s-%(name)s-%(levelname)s-[%(filename)s:%(lineno)d]-%(message)s"
)
es_handler.setFormatter(formatter2)
logging.getLogger().addHandler(es_handler)
# 禁止如下两个模块乱打调试日志，因为ELK会把 debug 日志打在屏幕，有干扰
logging.getLogger("elasticsearch").setLevel(logging.WARNING)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
# 创建 mysql 的工厂
if startmysql:
cls.mysqlengine = await create_engine(user=cls.mysqluser,
password=cls.mysqlpassword,
host=cls.mysqlhost,
port=cls.mysqlport,
db=cls.mysqldb,
loop=cls.asyncioloop)
if startpg:
cls.pg_pool = await asyncpg.create_pool(
user='******',
password='******',
database='goldhonor',
host='factory.goldhonor.com',
port=15432,
command_timeout=60)
pass
class ElasticSearchBackend(AbstractBackend):
"""
ElasticsearchBackend is used to create an abstraction of the handler and the formatter concepts from logging.
Here, we have a CMRESHandler which is part of an external library named 'cmreslogging' based on 'logging'.
CMRESHandler is a specific handler created to send log records to an ElasticSearch DB. It does not need a Formatter
object.
"""
def __init__(self):
"""
CMRESHandler needs, at least, a hostname, a username, a password, a port and a specific index
from the ElasticSearch DB to send log records.
"""
super(ElasticSearchBackend, self).__init__(None, None)
self.__host = ''
self.__user = None
self.__passwd = None
self.__port = 9203
self.__index = ''
self.__bufferSize = 1000
self.__flushTime = 1
def createHandler(self, parameters=None):
"""
Each backend can initialize its attributes and create its handler with them.
:params parameters: dictionary of parameters. ex: {'FileName': file.log}
"""
if parameters is not None:
self.__host = parameters.get('Host', self.__host)
self.__user = parameters.get('User', self.__user)
self.__passwd = parameters.get('Password', self.__passwd)
self.__port = int(parameters.get('Port', self.__port))
self.__index = parameters.get('Index', self.__index)
self.__bufferSize = int(parameters.get('BufferSize', self.__bufferSize))
self.__flushTime = int(parameters.get('FlushTime', self.__flushTime))
if self.__user is not None and self.__passwd is not None:
self._handler = CMRESHandler(hosts=[{'host': self.__host, 'port': self.__port}],
auth_type=CMRESHandler.AuthType.BASIC_AUTH,
auth_details=(self.__user, self.__passwd),
es_index_name=self.__index,
use_ssl=True,
verify_ssl=True,
buffer_size=self.__bufferSize,
flush_frequency_in_sec=self.__flushTime)
else:
self._handler = CMRESHandler(hosts=[{'host': self.__host, 'port': self.__port}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=self.__index,
use_ssl=True,
verify_ssl=True,
buffer_size=self.__bufferSize,
flush_frequency_in_sec=self.__flushTime)
# We give a format containing only asctime to add the field in elasticsearch
# asctime is not created at the initialization of the LogRecords but built in the format process
self._handler.setFormatter(logging.Formatter('%(asctime)s'))
def setLevel(self, level):
"""
No possibility to set the level of the ElasticSearch handler.
It is not set by default so it can send all Log Records of all levels to ElasticSearch.
"""
pass
def setFormat(self, fmt, datefmt, options):
"""
Each backend give a format to their formatters and attach them to their handlers.
:params fmt: string representing the log format
:params datefmt: string representing the date format
:params component: string represented as "system/component"
:params options: dictionary of logging options. ex: {'Color': True}
"""
pass