def __new__(cls):
if not hasattr(cls, 'instance'):
cls.instance = super(Logger, cls).__new__(cls)
cls.instance.logger = logging.getLogger('MScThesis')
cls.instance.logger.addHandler(MongoHandler.to(db=Config.database.database, collection='log'))
cls.instance.logger.setLevel(logging.DEBUG)
return cls.instance
def configure_mongolog():
"""This method uses a sync connection via PyMongo because the Python's
:mod:`logging` module is blocking; however, we set the write concern to
``0`` for no expecting a result. You can set a different database by
setting up the ``logging_db_uri`` option."""
client = pymongo.MongoClient(options.logging_db_uri, w=0)
from mongolog.handlers import MongoHandler
col = client[options.logging_db_name][options.logging_db_collection]
logging.getLogger().addHandler(MongoHandler.to(collection=col))
def open_handler(self):
# set logging
try:
self.log = logging.getLogger(self.name)
self.log.addHandler(MongoHandler.to(db=self.dbinfo["db"],
collection='log', host=self.dbinfo["host"],
port=self.dbinfo["port"], username=self.dbinfo["user"],
password=self.dbinfo["passwd"]))
except:
print "Faild to open mongodb"
raise
def connect():
from mongolog.handlers import MongoHandler
from configurations import importer
import os
import logging
from hbproject import settings
import dotenv
dotenv.read_dotenv()
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'hbproject.settings')
os.environ.setdefault('DJANGO_CONFIGURATION', 'Development')
importer.install()
log = logging.getLogger('heliosburn')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler.to(db=settings.MONGODB_DATABASE['production'], collection='log'))
return log
def setup_logging(level=logging.DEBUG):
""" Setup centralized logging
All log messages will end-up in capped mongo collection
"""
log = logging.getLogger('flickr')
host = 'localhost'
try:
import settings
host = settings.MONGO['host']
except ImportError:
print >>sys.stderr, 'Settings file is not available. Using localhost.'
log.addHandler(MongoHandler.to(db='logging', collection='tools', host=host))
log.setLevel(level)
return log
def init_logger(log_name, log_level, email=True):
""" Creates logger instance with MongoHandler """
log = logging.getLogger(log_name)
if 'debug' in log_level.lower():
log.setLevel(logging.DEBUG)
elif 'warn' in log_level.lower():
log.setLevel(logging.WARNING)
elif 'error' in log_level.lower():
log.setLevel(logging.ERROR)
elif 'crit' in log_level.lower():
log.setLevel(logging.CRITICAL)
log.addHandler(MongoHandler.to(db='logs', collection='intrader'))
if email:
u, p = get_gmail_auth()
log.addHandler(
TlsSMTPHandler(('smtp.gmail.com', 587), u, [u],
'Intrader Logging Report', (u, p)))
return log
def connect():
from mongolog.handlers import MongoHandler
from configurations import importer
import os
import logging
from hbproject import settings
import dotenv
dotenv.read_dotenv()
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'hbproject.settings')
os.environ.setdefault('DJANGO_CONFIGURATION', 'Development')
importer.install()
log = logging.getLogger('heliosburn')
log.setLevel(logging.DEBUG)
log.addHandler(
MongoHandler.to(db=settings.MONGODB_DATABASE['production'],
collection='log'))
return log
def get_mongohandler(collection,
db='mongolog',
host='localhost',
port=None,
username=None,
password=None,
level=logging.NOTSET):
"""获取mongodb日志处理器
:param collection 数据集
:param db 数据库
:param host 连接地址
:param port 端口 默认27017
:param username 登录名
:param password 密码
:param level 日志级别
"""
from mongolog.handlers import MongoHandler
hndr = MongoHandler.to(collection, db, host, port, username, password,
level)
return hndr
def init_logger(log_name, log_level, email = True):
""" Creates logger instance with MongoHandler """
log = logging.getLogger(log_name)
if 'debug' in log_level.lower():
log.setLevel(logging.DEBUG)
elif 'warn' in log_level.lower():
log.setLevel(logging.WARNING)
elif 'error' in log_level.lower():
log.setLevel(logging.ERROR)
elif 'crit' in log_level.lower():
log.setLevel(logging.CRITICAL)
log.addHandler(MongoHandler.to(db='logs', collection='intrader'))
if email:
u, p = get_gmail_auth()
log.addHandler(TlsSMTPHandler(('smtp.gmail.com', 587),
u,
[u],
'Intrader Logging Report',
(u, p)))
return log
def getMongoLogger(name, level='INFO', process_stage='default'):
"""
name = __name__
level = 'DEBUG' ou 'INFO' ou 'WARNING' ou 'ERROR' ou 'CRITICAL'
process_stage = 'extract' ou 'tranform' ou 'load' ou 'default'
"""
logger = logging.getLogger(name)
allowed_levels = {
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'CRITICAL': logging.CRITICAL
}
logging_level = allowed_levels.get(level, config.OPAC_PROC_LOG_LEVEL)
logger.setLevel(logging_level)
mongo_settings = config.OPAC_PROC_LOG_MONGODB_SETTINGS
mongo_settings['collection'] = "%s_log" % process_stage
mongo_handler = MongoHandler.to(**mongo_settings)
logger.addHandler(mongo_handler)
return logger
import sys
sys.path.append("..")
import logging
from pymongo.connection import Connection
from mongolog.handlers import MongoHandler
if __name__ == "__main__":
log = logging.getLogger("example")
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler.to("mongolog", "log"))
log.debug("1 - debug message")
log.info("2 - info message")
log.warn("3 - warn message")
log.error("4 - error message")
log.critical("5 - critical message")
import logging
from mongolog.handlers import MongoHandler
log = logging.getLogger('demo')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler.to(db='mongolog', collection='log'))
log.debug("1 - debug message")
log.info("2 - info message")
log.warn("3 - warn message")
log.error("4 - error message")
log.critical("5 - critical message")
import sys
sys.path.append('..')
import logging
from pymongo.connection import Connection
from mongolog.handlers import MongoHandler
if __name__ == '__main__':
log = logging.getLogger('example')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler.to('mongolog', 'log'))
log.debug("1 - debug message")
log.info("2 - info message")
log.warn("3 - warn message")
log.error("4 - error message")
log.critical("5 - critical message")
try:
raise Exception("6 - exception message")
except Exception:
log.exception("6 - exception message")
def __init__(self, **kwargs):
super(BaseSpider, self).__init__(**kwargs)
grab_log = logging.getLogger('grab')
grab_log.addHandler(MongoHandler.to(db=settings.MONGO_DB, collection='logs'))
from state_tool import after_get_parameter, get_need_verify_types, \
get_next_verify_action, generate_log_data,\
SUCCESS_STATUS, CRAWL_RETRY_TIMES
from setting.db_config import DB_CONFIG
from setting.status_code import STATUS_CODE
from worker.communicate import get_parameter, success_data, \
save_data, save_status, get_today_result
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
db_config = DB_CONFIG['state_log']
logger.addHandler(
MongoHandler.to(host=db_config['host'],
port=db_config['port'],
db=db_config['db'],
collection=db_config['collection']))
def extend_log_func(state):
result = {'verify': {}}
result['verify']['need_verify_request'] = state.need_verify_request
result['verify']['sent_verify_request'] = state.sent_verify_request
result['verify']['now_verify_request'] = ''
return result
class StartState(BaseState):
def __init__(self, **kwargs):
super(StartState, self).__init__(**kwargs)
self.parameters['tel'] = kwargs['tel']
import sys
sys.path.append('..')
import logging
from pymongo.connection import Connection
from mongolog.handlers import MongoHandler
if __name__ == '__main__':
log = logging.getLogger('example')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler.to('mongolog', 'log'))
log.debug("1 - debug message")
log.info("2 - info message")
log.warn("3 - warn message")
log.error("4 - error message")
log.critical("5 - critical message")
#!/usr/bin/env python # -*- coding: utf-8 - from config import * from mongolog.handlers import MongoHandler import logging logging.getLogger().setLevel(logging.DEBUG) logging.getLogger().addHandler(MongoHandler.to(host=MONGO_URL, port=MONGO_PORT, db=MONGO_DATABASE, collection='log'))
level=logging.DEBUG,
format=
'%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S',
filename='myapp.log',
filemode='w')
#################################################################################################
#定义一个StreamHandler，将INFO级别或更高的日志信息打印到标准错误，并将其添加到当前的日志处理对象#
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
#################################################################################################
logging.debug('This is debug message')
logging.info('This is info message')
logging.warning('This is warning message')
import logging
from mongolog.handlers import MongoHandler
log = logging.getLogger('demo')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler.to(db='mongolog', collection='log'))
log.debug("1 - debug message")
log.info("2 - info message")
log.warn("3 - warn message")
log.error("4 - error message")
log.critical("5 - critical message")
import logging
import os
from mongolog.handlers import MongoHandler
# Add API logger instance
log = logging.getLogger("api_log")
log.setLevel(logging.INFO)
log.addHandler(
MongoHandler.to(collection=os.getenv("API_LOG_COLLECTION", "request"),
db=os.getenv("API_LOG_DB", "api_log"),
host=os.getenv("API_LOG_HOST", "localhost"),
port=os.getenv("API_LOG_PORT", None),
username=os.getenv("API_LOG_USERNAME", None),
password=os.getenv("API_LOG_PASSWORD", None)))
def configure_mongolog():
client = pymongo.MongoClient(options.logging_db_uri, w=0)
from mongolog.handlers import MongoHandler
col = client[options.logging_db_name][options.logging_db_collection]
logging.getLogger().addHandler(MongoHandler.to(collection=col))
def get_logger(app,
name,
log_level=None,
log_file=None,
mongo_host=None,
mongo_port=None,
mongo_db=None,
mongo_collection=None):
global logger
if logger: return logger
# initialize flask access log
werkzeug_logger = logging.getLogger('werkzeug')
# initialize logging log_level
if not log_level:
log_level = logging.DEBUG
else:
log_level = get_log_level(log_level)
# initialize logger
logger = logging.getLogger(name)
logger.setLevel(log_level)
# initialize stream log
stream_handler = logging.StreamHandler()
stream_handler.setLevel(log_level)
stream_formatter = ColoredFormatter(
'%(log_color)s%(asctime)s %(white)s[pid:%(process)d] [%(levelname)s] (%(pathname)s:%(lineno)d) %(message)s',
reset=True,
log_colors={
'DEBUG': 'green',
'INFO': 'yellow',
'WARNING': 'cyan',
'ERROR': 'red',
'CRITICAL':'red',
})
stream_handler.setFormatter(stream_formatter)
logger.addHandler(stream_handler)
# initialize file log
if log_file:
file_handler = logging.FileHandler(log_file)
file_handler.setLevel(log_level)
file_formatter = Formatter('%(asctime)s;%(levelname)s;%(pathname)s:%(lineno)d;%(message)s')
file_handler.setFormatter(file_formatter)
logger.addHandler(file_handler)
werkzeug_logger.addHandler(file_handler)
app.logger.addHandler(file_handler)
# initialize remote log
if mongo_collection and mongo_db and mongo_host and mongo_port:
mongo_handler = MongoHandler.to(db=mongo_db,
collection=mongo_collection,
host=mongo_host,
port=mongo_port,
level=log_level)
logger.addHandler(mongo_handler)
return logger