def get_logentries_handler():
import os
from logentries import LogentriesHandler
handler = LogentriesHandler(os.getenv("LOGENTRIES_TOKEN"))
handler.setFormatter(formatter)
handler.setLevel(logging.INFO)
return handler
class LogentriesExtension(object):
def __init__(self, token):
self.token = token
root = logging.getLogger()
self.handler = LogentriesHandler(token)
spider_id = os.environ.get('SCRAPY_SPIDER_ID')
project_id = os.environ.get('SCRAPY_PROJECT_ID')
job_id = os.environ.get('SCRAPY_JOB_ID')
formatted = False
if job_id is not None:
formatted = True
filter = ScrapingHubFilter({
'project_id': project_id,
'spider_id': spider_id,
'job_id': job_id,
})
format = "%(name)s - %(levelname)s - [project_id=%(project_id)s spider_id=%(spider_id)s job_id=%(job_id)s] %(message)s"
formatter = logging.Formatter(format)
self.handler.addFilter(filter)
self.handler.setFormatter(formatter)
root.addHandler(self.handler)
# NCA: not sure we want sensitive information like the token in the logs
# Maybe use debug log level instead
if formatted:
logger.info(
'Logentries activated with token {} and custom SH format'.
format(token))
else:
logger.info(
'Logentries activated with token {} and no custom SH format'.
format(token))
@classmethod
def from_crawler(cls, crawler):
# first check if the extension should be enabled and raise
# NotConfigured otherwise
token = crawler.settings.get('LOGENTRIES_TOKEN')
if not token:
raise NotConfigured
# instantiate the extension object
ext = cls(token)
# return the extension object
return ext
class LogentriesExtension(object):
def __init__(self, token):
self.token = token
root = logging.getLogger()
self.handler = LogentriesHandler(token)
spider_id = os.environ.get('SCRAPY_SPIDER_ID')
project_id = os.environ.get('SCRAPY_PROJECT_ID')
job_id = os.environ.get('SCRAPY_JOB_ID')
formatted = False
if job_id is not None:
formatted = True
filter = ScrapingHubFilter({
'project_id': project_id,
'spider_id': spider_id,
'job_id': job_id,
})
format = "%(name)s - %(levelname)s - [project_id=%(project_id)s spider_id=%(spider_id)s job_id=%(job_id)s] %(message)s"
formatter = logging.Formatter(format)
self.handler.addFilter(filter)
self.handler.setFormatter(formatter)
root.addHandler(self.handler)
# NCA: not sure we want sensitive information like the token in the logs
# Maybe use debug log level instead
if formatted:
logger.info('Logentries activated with token {} and custom SH format'.format(token))
else:
logger.info('Logentries activated with token {} and no custom SH format'.format(token))
@classmethod
def from_crawler(cls, crawler):
# first check if the extension should be enabled and raise
# NotConfigured otherwise
token = crawler.settings.get('LOGENTRIES_TOKEN')
if not token:
raise NotConfigured
# instantiate the extension object
ext = cls(token)
# return the extension object
return ext
def init():
'''
Registers the msg-logger
'''
handler = None
if os.environ.get('LOGENTRIES_TOKEN') is not None:
handler = LogentriesHandler(os.environ.get('LOGENTRIES_TOKEN'),
format=CustomFormatter())
else:
handler = logging.StreamHandler()
handler.setFormatter(CustomFormatter())
logger = logging.getLogger('msg-logger')
logger.setLevel(logging.INFO)
logger.addFilter(CustomFilter())
logger.addHandler(handler)
def setup_logging(loggers_and_levels, logentries_id=None):
log = logging.getLogger('logentries')
log.setLevel(logging.INFO)
logentries_handler = LogentriesHandler(logentries_id)
handler = logging.StreamHandler()
FORMAT = "%(asctime)s:%(levelname)s:%(name)s:%(message)s"
formatter = logging.Formatter(fmt=FORMAT)
handler.setFormatter(formatter)
logentries_handler.setFormatter(formatter)
log.addHandler(handler)
log.addHandler(logentries_handler)
for logger, level in loggers_and_levels:
logger.setLevel(level)
logger.addHandler(handler)
logger.addHandler(logentries_handler)
def get_custom_logger(name):
""" Set up loggers according to environment and configuration. """
file = app.config['LOGGER_FILEPATH']
token = app.config['LOGENTRIES_TOKEN']
level = DEBUG if app.config['DEBUG'] else INFO
context = app.config['ENVIRONMENT']
log = getLogger(name)
log.setLevel(level)
# shared by all handlers
formatter = Formatter(
'[%(asctime)s] '
'[%(process)d] '
'[%(name)s] '
'[%(levelname)s] '
'%(message)s'
)
if context != 'testing' and file:
file = FileHandler(file, mode='a+')
file.setLevel(level)
file.setFormatter(formatter)
log.addHandler(file)
if context == 'development':
console = StreamHandler(stream=stdout)
console.setLevel(level)
console.setFormatter(formatter)
log.addHandler(console)
if context != 'testing' and token:
logentries = LogentriesHandler(token)
logentries.setLevel(level)
logentries.setFormatter(formatter)
log.addHandler(logentries)
return log
syslog_formatter = logging.Formatter('%(asctime)s ' + syslog_hostname +
' %(name)s: %(message)s',
datefmt='%b %d %H:%M:%S')
syslog_address = os.getenv('SYSLOG_ADDRESS', '')
if syslog_address:
syslog_host, syslog_udp_port = syslog_address.split(":")
syslog_handler = SysLogHandler(address=(syslog_host, int(syslog_udp_port)))
syslog_handler.setFormatter(syslog_formatter)
syslog_handler.setLevel(syslog_logging_level)
logging_handlers.append(syslog_handler)
logentries_token = os.getenv('LOGENTRIES_TOKEN', '')
if logentries_token:
logentries_handler = LogentriesHandler(logentries_token)
logentries_handler.setFormatter(syslog_formatter)
logentries_handler.setLevel(syslog_logging_level)
logging_handlers.append(logentries_handler)
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.DEBUG,
handlers=logging_handlers)
def main():
botan_token = os.getenv('BOTAN_TOKEN', '')
sc_auth_token = os.environ['SC_AUTH_TOKEN']
store_chat_id = int(os.getenv('STORE_CHAT_ID', '0'))
no_flood_chat_ids = list(
request.remote_addr)
if current_user.is_anonymous():
log_record.user_id = 'guest'
else:
log_record.user_id = current_user.get_id()
return True
# Use said info
log_format = ("%(utcnow)s\tl=%(levelname)s\tu=%(user_id)s\tip=%(ip)s"
"\tm=%(method)s\turl=%(url)s\tmsg=%(message)s")
formatter = logging.Formatter(log_format)
# - Handlers
# -- Stream handler
streamHandler = logging.StreamHandler()
streamHandler.setLevel(logging.INFO)
streamHandler.setFormatter(formatter)
# -- LogEntries handler
leHandler = LogentriesHandler(app.config['LOGENTRIES_TOKEN'])
leHandler.setLevel(logging.INFO)
leHandler.setFormatter(formatter)
# - Logger
log = app.logger
log.setLevel(logging.DEBUG)
log.addFilter(ContextualFilter())
log.addHandler(streamHandler)
log.addHandler(leHandler)
