def __new__(cls):
if not hasattr(cls, 'instance'):
cls.instance = super(Logger, cls).__new__(cls)
cls.instance.logger = logging.getLogger('MScThesis')
cls.instance.logger.addHandler(MongoHandler.to(db=Config.database.database, collection='log'))
cls.instance.logger.setLevel(logging.DEBUG)
return cls.instance
def configure_mongolog():
"""This method uses a sync connection via PyMongo because the Python's
:mod:`logging` module is blocking; however, we set the write concern to
``0`` for no expecting a result. You can set a different database by
setting up the ``logging_db_uri`` option."""
client = pymongo.MongoClient(options.logging_db_uri, w=0)
from mongolog.handlers import MongoHandler
col = client[options.logging_db_name][options.logging_db_collection]
logging.getLogger().addHandler(MongoHandler.to(collection=col))
def testLogging(self):
""" Simple logging example """
log = logging.getLogger('')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler(self.collection))
log.debug('test')
r = self.collection.find_one({'levelname': 'DEBUG', 'msg': 'test'})
self.assertEquals(r['msg'], 'test')
def open_handler(self):
# set logging
try:
self.log = logging.getLogger(self.name)
self.log.addHandler(MongoHandler.to(db=self.dbinfo["db"],
collection='log', host=self.dbinfo["host"],
port=self.dbinfo["port"], username=self.dbinfo["user"],
password=self.dbinfo["passwd"]))
except:
print "Faild to open mongodb"
raise
def testLoggingException(self):
""" Logging example with exception """
log = logging.getLogger('')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler(self.collection))
try:
1/0
except ZeroDivisionError:
log.error('test zero division', exc_info=True)
r = self.collection.find_one({'levelname':'ERROR', 'msg':'test zero division'})
self.assertTrue(r['exc_info'].startswith('Traceback'))
def setUp(self):
""" Create an empty database that could be used for logging """
self.db_name = "_mongolog_test"
self.conn = Connection("localhost")
self.conn.drop_database(self.db_name)
self.db = self.conn[self.db_name]
self.collection = self.db["log"]
self.log = logging.getLogger("")
self.log.setLevel(logging.DEBUG)
self.handler = MongoHandler(self.collection)
self.log.addHandler(self.handler)
def setup_logging(level=logging.DEBUG):
""" Setup centralized logging
All log messages will end-up in capped mongo collection
"""
log = logging.getLogger('flickr')
host = 'localhost'
try:
import settings
host = settings.MONGO['host']
except ImportError:
print >>sys.stderr, 'Settings file is not available. Using localhost.'
log.addHandler(MongoHandler.to(db='logging', collection='tools', host=host))
log.setLevel(level)
return log
def connect():
from mongolog.handlers import MongoHandler
from configurations import importer
import os
import logging
from hbproject import settings
import dotenv
dotenv.read_dotenv()
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'hbproject.settings')
os.environ.setdefault('DJANGO_CONFIGURATION', 'Development')
importer.install()
log = logging.getLogger('heliosburn')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler.to(db=settings.MONGODB_DATABASE['production'], collection='log'))
return log
def testQueryableMessages(self):
""" Logging example with dictionary """
log = logging.getLogger('query')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler(self.collection))
log.info({'address': '340 N 12th St', 'state': 'PA', 'country': 'US'})
log.info({'address': '340 S 12th St', 'state': 'PA', 'country': 'US'})
log.info({'address': '1234 Market St', 'state': 'PA', 'country': 'US'})
cursor = self.collection.find({'level':'info', 'msg.address': '340 N 12th St'})
self.assertEquals(cursor.count(), 1, "Expected query to return 1 "
"message; it returned %d" % cursor.count())
self.assertEquals(cursor[0]['msg']['address'], '340 N 12th St')
cursor = self.collection.find({'level':'info', 'msg.state': 'PA'})
self.assertEquals(cursor.count(), 3, "Didn't find all three documents")
def connect():
from mongolog.handlers import MongoHandler
from configurations import importer
import os
import logging
from hbproject import settings
import dotenv
dotenv.read_dotenv()
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'hbproject.settings')
os.environ.setdefault('DJANGO_CONFIGURATION', 'Development')
importer.install()
log = logging.getLogger('heliosburn')
log.setLevel(logging.DEBUG)
log.addHandler(
MongoHandler.to(db=settings.MONGODB_DATABASE['production'],
collection='log'))
return log
def init_logger(log_name, log_level, email=True):
""" Creates logger instance with MongoHandler """
log = logging.getLogger(log_name)
if 'debug' in log_level.lower():
log.setLevel(logging.DEBUG)
elif 'warn' in log_level.lower():
log.setLevel(logging.WARNING)
elif 'error' in log_level.lower():
log.setLevel(logging.ERROR)
elif 'crit' in log_level.lower():
log.setLevel(logging.CRITICAL)
log.addHandler(MongoHandler.to(db='logs', collection='intrader'))
if email:
u, p = get_gmail_auth()
log.addHandler(
TlsSMTPHandler(('smtp.gmail.com', 587), u, [u],
'Intrader Logging Report', (u, p)))
return log
def get_mongohandler(collection,
db='mongolog',
host='localhost',
port=None,
username=None,
password=None,
level=logging.NOTSET):
"""获取mongodb日志处理器
:param collection 数据集
:param db 数据库
:param host 连接地址
:param port 端口 默认27017
:param username 登录名
:param password 密码
:param level 日志级别
"""
from mongolog.handlers import MongoHandler
hndr = MongoHandler.to(collection, db, host, port, username, password,
level)
return hndr
def init_logger(log_name, log_level, email = True):
""" Creates logger instance with MongoHandler """
log = logging.getLogger(log_name)
if 'debug' in log_level.lower():
log.setLevel(logging.DEBUG)
elif 'warn' in log_level.lower():
log.setLevel(logging.WARNING)
elif 'error' in log_level.lower():
log.setLevel(logging.ERROR)
elif 'crit' in log_level.lower():
log.setLevel(logging.CRITICAL)
log.addHandler(MongoHandler.to(db='logs', collection='intrader'))
if email:
u, p = get_gmail_auth()
log.addHandler(TlsSMTPHandler(('smtp.gmail.com', 587),
u,
[u],
'Intrader Logging Report',
(u, p)))
return log
def getMongoLogger(name, level='INFO', process_stage='default'):
"""
name = __name__
level = 'DEBUG' ou 'INFO' ou 'WARNING' ou 'ERROR' ou 'CRITICAL'
process_stage = 'extract' ou 'tranform' ou 'load' ou 'default'
"""
logger = logging.getLogger(name)
allowed_levels = {
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'CRITICAL': logging.CRITICAL
}
logging_level = allowed_levels.get(level, config.OPAC_PROC_LOG_LEVEL)
logger.setLevel(logging_level)
mongo_settings = config.OPAC_PROC_LOG_MONGODB_SETTINGS
mongo_settings['collection'] = "%s_log" % process_stage
mongo_handler = MongoHandler.to(**mongo_settings)
logger.addHandler(mongo_handler)
return logger
def configure_mongolog():
client = pymongo.MongoClient(options.logging_db_uri, w=0)
from mongolog.handlers import MongoHandler
col = client[options.logging_db_name][options.logging_db_collection]
logging.getLogger().addHandler(MongoHandler.to(collection=col))
level=logging.DEBUG,
format=
'%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S',
filename='myapp.log',
filemode='w')
#################################################################################################
#定义一个StreamHandler，将INFO级别或更高的日志信息打印到标准错误，并将其添加到当前的日志处理对象#
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
#################################################################################################
logging.debug('This is debug message')
logging.info('This is info message')
logging.warning('This is warning message')
import logging
from mongolog.handlers import MongoHandler
log = logging.getLogger('demo')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler.to(db='mongolog', collection='log'))
log.debug("1 - debug message")
log.info("2 - info message")
log.warn("3 - warn message")
log.error("4 - error message")
log.critical("5 - critical message")
import sys
sys.path.append("..")
import logging
from pymongo.connection import Connection
from mongolog.handlers import MongoHandler
if __name__ == "__main__":
log = logging.getLogger("example")
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler.to("mongolog", "log"))
log.debug("1 - debug message")
log.info("2 - info message")
log.warn("3 - warn message")
log.error("4 - error message")
log.critical("5 - critical message")
import logging
from mongolog.handlers import MongoHandler
log = logging.getLogger('demo')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler.to(db='mongolog', collection='log'))
log.debug("1 - debug message")
log.info("2 - info message")
log.warn("3 - warn message")
log.error("4 - error message")
log.critical("5 - critical message")
import sys
sys.path.append('..')
import logging
from pymongo.connection import Connection
from mongolog.handlers import MongoHandler
if __name__ == '__main__':
log = logging.getLogger('example')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler.to('mongolog', 'log'))
log.debug("1 - debug message")
log.info("2 - info message")
log.warn("3 - warn message")
log.error("4 - error message")
log.critical("5 - critical message")
try:
raise Exception("6 - exception message")
except Exception:
log.exception("6 - exception message")
def __init__(self, **kwargs):
super(BaseSpider, self).__init__(**kwargs)
grab_log = logging.getLogger('grab')
grab_log.addHandler(MongoHandler.to(db=settings.MONGO_DB, collection='logs'))
from state_tool import after_get_parameter, get_need_verify_types, \
get_next_verify_action, generate_log_data,\
SUCCESS_STATUS, CRAWL_RETRY_TIMES
from setting.db_config import DB_CONFIG
from setting.status_code import STATUS_CODE
from worker.communicate import get_parameter, success_data, \
save_data, save_status, get_today_result
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
db_config = DB_CONFIG['state_log']
logger.addHandler(
MongoHandler.to(host=db_config['host'],
port=db_config['port'],
db=db_config['db'],
collection=db_config['collection']))
def extend_log_func(state):
result = {'verify': {}}
result['verify']['need_verify_request'] = state.need_verify_request
result['verify']['sent_verify_request'] = state.sent_verify_request
result['verify']['now_verify_request'] = ''
return result
class StartState(BaseState):
def __init__(self, **kwargs):
super(StartState, self).__init__(**kwargs)
self.parameters['tel'] = kwargs['tel']
import sys
sys.path.append('..')
import logging
from pymongo.connection import Connection
from mongolog.handlers import MongoHandler
if __name__ == '__main__':
log = logging.getLogger('example')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler.to('mongolog', 'log'))
log.debug("1 - debug message")
log.info("2 - info message")
log.warn("3 - warn message")
log.error("4 - error message")
log.critical("5 - critical message")
def get_logger(app,
name,
log_level=None,
log_file=None,
mongo_host=None,
mongo_port=None,
mongo_db=None,
mongo_collection=None):
global logger
if logger: return logger
# initialize flask access log
werkzeug_logger = logging.getLogger('werkzeug')
# initialize logging log_level
if not log_level:
log_level = logging.DEBUG
else:
log_level = get_log_level(log_level)
# initialize logger
logger = logging.getLogger(name)
logger.setLevel(log_level)
# initialize stream log
stream_handler = logging.StreamHandler()
stream_handler.setLevel(log_level)
stream_formatter = ColoredFormatter(
'%(log_color)s%(asctime)s %(white)s[pid:%(process)d] [%(levelname)s] (%(pathname)s:%(lineno)d) %(message)s',
reset=True,
log_colors={
'DEBUG': 'green',
'INFO': 'yellow',
'WARNING': 'cyan',
'ERROR': 'red',
'CRITICAL':'red',
})
stream_handler.setFormatter(stream_formatter)
logger.addHandler(stream_handler)
# initialize file log
if log_file:
file_handler = logging.FileHandler(log_file)
file_handler.setLevel(log_level)
file_formatter = Formatter('%(asctime)s;%(levelname)s;%(pathname)s:%(lineno)d;%(message)s')
file_handler.setFormatter(file_formatter)
logger.addHandler(file_handler)
werkzeug_logger.addHandler(file_handler)
app.logger.addHandler(file_handler)
# initialize remote log
if mongo_collection and mongo_db and mongo_host and mongo_port:
mongo_handler = MongoHandler.to(db=mongo_db,
collection=mongo_collection,
host=mongo_host,
port=mongo_port,
level=log_level)
logger.addHandler(mongo_handler)
return logger
class TestRootLoggerHandler(unittest.TestCase):
"""
Test Handler attached to RootLogger
"""
def setUp(self):
""" Create an empty database that could be used for logging """
self.db_name = "_mongolog_test"
self.conn = Connection("localhost")
self.conn.drop_database(self.db_name)
self.db = self.conn[self.db_name]
self.collection = self.db["log"]
self.log = logging.getLogger("")
self.log.setLevel(logging.DEBUG)
self.handler = MongoHandler(self.collection)
self.log.addHandler(self.handler)
def tearDown(self):
""" Drop used database """
self.conn.drop_database(self.db_name)
self.log.removeHandler(self.handler)
def testLogging(self):
""" Simple logging example """
self.log.debug("test")
r = self.collection.find_one({"levelname": "DEBUG", "msg": "test"})
self.assertEquals(r["msg"], "test")
def testLoggingException(self):
""" Logging example with exception """
try:
1 / 0
except ZeroDivisionError:
self.log.error("test zero division", exc_info=True)
r = self.collection.find_one({"levelname": "ERROR", "msg": "test zero division"})
self.assertTrue(r["exc_info"].startswith("Traceback"))
def testQueryableMessages(self):
""" Logging example with dictionary """
self.log.info({"address": "340 N 12th St", "state": "PA", "country": "US"})
self.log.info({"address": "340 S 12th St", "state": "PA", "country": "US"})
self.log.info({"address": "1234 Market St", "state": "PA", "country": "US"})
cursor = self.collection.find({"levelname": "INFO", "msg.address": "340 N 12th St"})
self.assertEquals(cursor.count(), 1, "Expected query to return 1 " "message; it returned %d" % cursor.count())
self.assertEquals(cursor[0]["msg"]["address"], "340 N 12th St")
cursor = self.collection.find({"levelname": "INFO", "msg.state": "PA"})
self.assertEquals(cursor.count(), 3, "Didn't find all three documents")
def testFormatter(self):
formatString = "%(message)s from %(levelname)s"
self.handler.setFormatter(MongoFormatter(formatString))
self.log.info("%s within a message", "message")
document = self.collection.find_one()
self.assertEquals(document["message"], "message within a message from" " INFO")
def testNoneArgs(self):
""" Logging example with "None" as logging args """
self.log.info("This is a string %s with no args", None)
#!/usr/bin/env python # -*- coding: utf-8 - from config import * from mongolog.handlers import MongoHandler import logging logging.getLogger().setLevel(logging.DEBUG) logging.getLogger().addHandler(MongoHandler.to(host=MONGO_URL, port=MONGO_PORT, db=MONGO_DATABASE, collection='log'))
import logging
import os
from mongolog.handlers import MongoHandler
# Add API logger instance
log = logging.getLogger("api_log")
log.setLevel(logging.INFO)
log.addHandler(
MongoHandler.to(collection=os.getenv("API_LOG_COLLECTION", "request"),
db=os.getenv("API_LOG_DB", "api_log"),
host=os.getenv("API_LOG_HOST", "localhost"),
port=os.getenv("API_LOG_PORT", None),
username=os.getenv("API_LOG_USERNAME", None),
password=os.getenv("API_LOG_PASSWORD", None)))