def setup_spider_logging(spider, settings): """Initialize and configure default loggers Copied from Scrapy and updated, because version from Scrapy: 1) doesn't close handlers and observers 2) opens logobserver for twisted logging each time it's called - you can find N log observers logging the same message N after N crawls. so there's no way to reuse it. :return: method that should be called to cleanup handler. """ if isinstance(settings, dict): settings = Settings(settings) # Looging stdout is a bad idea when mutiple crawls are running # if settings.getbool('LOG_STDOUT'): # sys.stdout = StreamLogger(logging.getLogger('stdout')) filename = settings.get('LOG_FILE') if filename: encoding = settings.get('LOG_ENCODING') handler = logging.FileHandler(filename, encoding=encoding) elif settings.getbool('LOG_ENABLED'): handler = logging.StreamHandler() else: handler = logging.NullHandler() formatter = logging.Formatter(fmt=settings.get('LOG_FORMAT'), datefmt=settings.get('LOG_DATEFORMAT')) handler.setFormatter(formatter) handler.setLevel(settings.get('LOG_LEVEL')) filters = [ TopLevelFormatter(['scrapy']), SpiderFilter(spider), ] for _filter in filters: handler.addFilter(_filter) logging.root.addHandler(handler) _cleanup_functions = [ lambda: [handler.removeFilter(f) for f in filters], lambda: logging.root.removeHandler(handler), handler.close, ] def cleanup(): for func in _cleanup_functions: try: func() except Exception as e: err(e) return cleanup
def setUp(self): self.handler = LogCapture() self.handler.addFilter(TopLevelFormatter(['test']))