def setup_log(cls):
'''
Logs access requests to gramex.requests as a CSV file.
'''
logger = logging.getLogger('gramex.requests')
keys = objectpath(conf, 'log.handlers.requests.keys', [])
log_info = build_log_info(keys)
cls.log_request = lambda handler: logger.info(log_info(handler))
def gramexlog(conf):
'''Set up gramexlog service'''
from gramex.transforms import build_log_info
try:
from elasticsearch import Elasticsearch, helpers
except ImportError:
app_log.error(
'gramexlog: elasticsearch missing. pip install elasticsearch')
return
# We call push() every 'flush' seconds on the main IOLoop. Defaults to every 5 seconds
flush = conf.pop('flush', 5)
ioloop = info._main_ioloop or tornado.ioloop.IOLoop.current()
# Set the defaultapp to the first config key under gramexlog:
if len(conf):
info.gramexlog.defaultapp = next(iter(conf.keys()))
for app, app_conf in conf.items():
app_config = info.gramexlog.apps[app] = AttrDict()
app_config.queue = []
keys = app_conf.pop('keys', [])
# If user specifies keys: [port, args.x, ...], these are captured as additional keys.
# The keys use same spec as Gramex logging.
app_config.extra_keys = build_log_info(keys)
# Ensure all gramexlog keys are popped from app_conf, leaving only Elasticsearch keys
app_config.conn = Elasticsearch(**app_conf)
def push():
for app, app_config in info.gramexlog.apps.items():
for item in app_config.queue:
item['_index'] = app_config.get('index', app)
try:
helpers.bulk(app_config.conn, app_config.queue)
app_config.queue.clear()
except Exception:
# TODO: If the connection broke, re-create it
# This generic exception should be caught for thread to continue its execution
app_log.exception('gramexlog: push to %s failed', app)
if 'handle' in info.gramexlog:
ioloop.remove_timeout(info.gramexlog.handle)
# Call again after flush seconds
info.gramexlog.handle = ioloop.call_later(flush, push)
info.gramexlog.handle = ioloop.call_later(flush, push)
info.gramexlog.push = push