def setup_spider_logging(spider, settings):
"""Initialize and configure default loggers
Copied from Scrapy and updated, because version from Scrapy:
1) doesn't close handlers and observers
2) opens logobserver for twisted logging each time it's called -
you can find N log observers logging the same message N
after N crawls.
so there's no way to reuse it.
:return: method that should be called to cleanup handler.
"""
if isinstance(settings, dict):
settings = Settings(settings)
# Looging stdout is a bad idea when mutiple crawls are running
# if settings.getbool('LOG_STDOUT'):
# sys.stdout = StreamLogger(logging.getLogger('stdout'))
filename = settings.get('LOG_FILE')
if filename:
encoding = settings.get('LOG_ENCODING')
handler = logging.FileHandler(filename, encoding=encoding)
elif settings.getbool('LOG_ENABLED'):
handler = logging.StreamHandler()
else:
handler = logging.NullHandler()
formatter = logging.Formatter(fmt=settings.get('LOG_FORMAT'),
datefmt=settings.get('LOG_DATEFORMAT'))
handler.setFormatter(formatter)
handler.setLevel(settings.get('LOG_LEVEL'))
filters = [
TopLevelFormatter(['scrapy']),
SpiderFilter(spider),
]
for _filter in filters:
handler.addFilter(_filter)
logging.root.addHandler(handler)
_cleanup_functions = [
lambda: [handler.removeFilter(f) for f in filters],
lambda: logging.root.removeHandler(handler),
handler.close,
]
def cleanup():
for func in _cleanup_functions:
try:
func()
except Exception as e:
err(e)
return cleanup
def setUp(self):
self.handler = LogCapture()
self.handler.addFilter(TopLevelFormatter(['test']))