def get_logger(self):
if self.logger == None:
format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=format)
formatter = logging.Formatter(format)
self.logger = logging.getLogger()
filehandler = RotatingFileHandler("./logs/bigcgi.log",
maxBytes=50000,
backupCount=2)
filehandler.setFormatter(formatter)
self.logger.addHandler(filehandler)
try:
handler = BufferedMongoHandler(
host='localhost',
capped=True,
database_name="bigcgi-logs",
username=AppSettings.DATABASE_USERNAME,
password=AppSettings.DATABASE_PASSWORD,
authentication_db="bigcgi-main",
buffer_size=100,
buffer_periodical_flush_timing=10.0,
buffer_early_flush_level=logging.CRITICAL,
connectTimeoutMS=5000,
serverSelectionTimeoutMS=5000,
)
self.logger.addHandler(handler)
except pymongo.errors.ServerSelectionTimeoutError:
print("Failed to connecto to MongoDB logger.")
sys.exit(1)
self.logger.setLevel("INFO")
return self.logger
def initialize():
# Creating capped handler
self.handler_periodical = BufferedMongoHandler(host=self.host_name,
database_name=self.database_name,
collection=self.collection_name,
buffer_size=5, buffer_periodical_flush_timing=2.0,
buffer_early_flush_level=logging.CRITICAL)
self.log.removeHandler(self.handler)
self.log.addHandler(self.handler_periodical)
def setUp(self):
self.handler = BufferedMongoHandler(host=self.host_name,
database_name=self.database_name,
collection=self.collection_name,
buffer_size=5, buffer_periodical_flush_timing=None,
buffer_early_flush_level=logging.CRITICAL
)
self.log = logging.getLogger('testLogger')
self.log.setLevel(logging.DEBUG)
self.log.addHandler(self.handler)
self.old_stderr = sys.stdout
sys.stderr = StringIO()
def log_factory(db_name):
mongo_host = os.environ.get('MONGO_URI')
handler = BufferedMongoHandler(host=mongo_host,
database_name=db_name,
collection='crawler_logs',
buffer_early_flush_level=logging.ERROR)
logger = logging.getLogger()
log = logging.getLogger('cardataLogger')
log.setLevel(logging.DEBUG)
log.handlers = []
log.addHandler(handler)
return log, handler
def get_handler():
'''
Return a log4mongo logging handler.
Use docker networking if in a docker container.
Otherwise use a mongodb that exists on the localhost
'''
if os.path.exists('/.dockerenv'):
host = 'mongo'
else:
host = 'localhost'
# Buffered handler writes logs periodically, to prevent frequent write-locks
handler = BufferedMongoHandler(host=host, port=27017,
buffer_size=1000,
buffer_periodical_flush_timing=10.0,
buffer_early_flush_level=logging.CRITICAL)
return handler
def test_buffer_periodical_flush(self):
# Creating capped handler
self.handler_periodical = BufferedMongoHandler(host=self.host_name,
database_name=self.database_name,
collection=self.collection_name,
buffer_size=5, buffer_periodical_flush_timing=2.0,
buffer_early_flush_level=logging.CRITICAL)
self.log.removeHandler(self.handler)
self.log.addHandler(self.handler_periodical)
self.log.info('test periodical buffer')
document = self.handler_periodical.collection.find_one({'message': 'test periodical buffer'})
self.assertIsNone(document, "Should not have been written to database") # except if your computer is really slow
self.assertEqual(len(self.handler_periodical.buffer), 1, "Buffer size should be 1")
self.log.info('test periodical buffer')
document = self.handler_periodical.collection.find_one({'message': 'test periodical buffer'})
self.assertIsNone(document, "Should not have been written to database")
self.assertEqual(len(self.handler_periodical.buffer), 2, "Buffer size should be 2")
time.sleep(2.5) # wait a bit so the periodical timer thread has kicked off the buffer flush
document = self.handler_periodical.collection.find_one({'message': 'test periodical buffer'})
self.assertIsNotNone(document, "Should not have been written to database")
doc_amount = self.handler_periodical.collection.find({'message': 'test periodical buffer'}).count()
self.assertEqual(doc_amount, 2, "Should have found 2 documents written to database")
self.assertEqual(len(self.handler_periodical.buffer), 0, "Buffer should be empty")
self.assertTrue(self.handler_periodical.buffer_timer_thread.is_alive())
self.handler_periodical.destroy()
time.sleep(0.2) # waiting a tiny bit so that the child thread actually exits
self.assertFalse(self.handler_periodical.buffer_timer_thread.is_alive(),
"Child timer thread should be dead by now. Slow computer?")
# reset to previous
self.log.removeHandler(self.handler_periodical)
self.log.addHandler(self.handler)
def getMongoLogHandler(loggerName=None, **kwargs):
"""
:param loggerName: logger名，整个项目应该共用一个名称，
这个很重要，为了后期按项目筛选日志
:param kwargs: 数据库的信息
:return:
"""
fmt = MongoFormatter(loggerName=loggerName)
mlh = BufferedMongoHandler(
# host='222.178.152.79',
# port=4316,
# database_name='origin',
# collection='org_logs',
# username='******',
# password='******',
# authSource='admin',
formatter=fmt,
# capped=True,
buffer_size=100, # buffer size.
buffer_periodical_flush_timing=10.0, # periodical flush every 10 seconds
buffer_early_flush_level=logging.CRITICAL, # early flush level
**kwargs
)
# mlh = MongoHandler(
# host='222.178.152.79',
# port=4316,
# database_name='origin',
# collection='org_logs',
# username='******',
# password='******',
# authSource='admin',
# formatter=fmt
# )
return mlh
def initialize_access_logger(app):
class AccessRecordFormatter(logging.Formatter):
def format(self, record):
remote_address, method, path, status = record.args
document = {
"timestamp": datetime.datetime.utcnow(),
"remote_address": remote_address,
"method": method,
"path": path,
"status": status
}
return document
access_handler = BufferedMongoHandler(
host=app.config["MONGO_HOST"],
port=app.config["MONGO_PORT"],
database_name=app.config["MONGO_DB"],
collection="access_log",
username=app.config["MONGO_USER"],
password=app.config["MONGO_PASSWORD"],
authentication_db=app.config["MONGO_DB"],
formatter=AccessRecordFormatter(),
buffer_size=100,
buffer_periodical_flush_timing=10.0,
buffer_early_flush_level=logging.CRITICAL,
)
access_logger = logging.getLogger("flask.app.access")
access_logger.setLevel(logging.INFO)
access_logger.addHandler(access_handler)
@app.after_request
def after_request(response):
logger = logging.getLogger("flask.app.access")
logger.info("", request.remote_addr, request.method, request.path,
response.status)
return response
from pymongo import MongoClient
from static import MONGODB, FILEINFO, EMAILINFO
from datetime import datetime
import logging
import logging.handlers
from log4mongo.handlers import BufferedMongoHandler
bmh = BufferedMongoHandler(host=MONGODB["host"], # All MongoHandler parameters are valid
port=MONGODB["port"],
username=MONGODB["user"],
password=MONGODB["passwd"],
capped=True,
buffer_size=100, # buffer size.
buffer_periodical_flush_timing=10.0, # periodical flush every 10 seconds
buffer_early_flush_level=logging.CRITICAL) # early flush
# create console handler and set level to debug
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter('%(asctime)-15s - %(levelname)s - %(collection_name)s : %(message)s')
# add formatter to ch
ch.setFormatter(formatter)
fh = logging.FileHandler(FILEINFO["loc"])
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
