def setup(): cfg = parse_config() logging.getLogger().setLevel(getattr(logging, cfg.get('log', 'loglevel').upper())) logfile = cfg.get('log', 'logfile') if logfile != '': handler = WatchedFileHandler(logfile) handler.addFilter(RequestIdFilter()) formatter = logging.Formatter( '%(asctime)s [%(process)d] %(levelname)-5s ' '%(request_id)s %(name)s %(message)s' ) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) else: logging.basicConfig() if cfg.has_option("metrics", "sentry_dsn"): # Only import sentry if enabled import sentry_sdk from sentry_sdk.integrations.flask import FlaskIntegration sentry_sdk.init( dsn=cfg.get("metrics", "sentry_dsn"), integrations=[FlaskIntegration()], ) if cfg.has_option("metrics", "prometheus_port"): prometheus_client.start_http_server( port=cfg.getint("metrics", "prometheus_port"), addr=cfg.get("metrics", "prometheus_addr"), ) ctx = SygnalContext() ctx.database = sygnal.db.Db(cfg.get('db', 'dbfile')) for key,val in cfg.items('apps'): parts = key.rsplit('.', 1) if len(parts) < 2: continue if parts[1] == 'type': try: pushkins[parts[0]] = make_pushkin(val, parts[0]) except: logger.exception("Failed to load module for kind %s", val) raise if len(pushkins) == 0: logger.error("No app IDs are configured. Edit sygnal.conf to define some.") sys.exit(1) for p in pushkins: pushkins[p].cfg = cfg pushkins[p].setup(ctx) logger.info("Configured with app IDs: %r", pushkins.keys()) logger.error("Setup completed")
def setup(): cfg = parse_config() logging.getLogger().setLevel(getattr(logging, cfg.get('log', 'loglevel').upper())) logfile = cfg.get('log', 'logfile') if logfile != '': handler = WatchedFileHandler(logfile) handler.addFilter(RequestIdFilter()) formatter = logging.Formatter( '%(asctime)s [%(process)d] %(levelname)-5s ' '%(request_id)s %(name)s %(message)s' ) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) else: logging.basicConfig() ctx = SygnalContext() ctx.database = sygnal.db.Db(cfg.get('db', 'dbfile')) for key,val in cfg.items('apps'): parts = key.rsplit('.', 1) if len(parts) < 2: continue if parts[1] == 'type': try: pushkins[parts[0]] = make_pushkin(val, parts[0]) except: logger.exception("Failed to load module for kind %s", val) raise if len(pushkins) == 0: logger.error("No app IDs are configured. Edit sygnal.conf to define some.") sys.exit(1) for p in pushkins: pushkins[p].cfg = cfg pushkins[p].setup(ctx) logger.info("Configured with app IDs: %r", pushkins.keys()) logger.error("Setup completed")
def generate_logger(clazz_name): logger = logging.getLogger(clazz_name) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s' ) handler = logging.StreamHandler() handler.setLevel(INFO) handler.setFormatter(formatter) logger.addHandler(handler) file_handler = WatchedFileHandler('log/app.log') file_handler.setLevel(INFO) file_handler.setFormatter(formatter) file_handler.addFilter(LoggingFilter(ERROR)) logger.addHandler(file_handler) logger.setLevel(INFO) logger.propagate = False return logger
def __init__( self, level=logging.DEBUG, name=None, logdir="./", stdout=True, multiFile=False, post=False, ws="www-qa.coraid.com", ): self.logdir = logdir self.ws = ws self.instance = os.environ.get("instance") or "" self.level = level logging.addLevelName(COMMENT, "COMMENT") # Root Logger self.logger = logging.getLogger("otto" + self.instance) self.logger.addHandler(logging.NullHandler()) """ Root Logger Threshold is WARNING by default. We will set the threshold as low as possible """ self.logger.setLevel(DEBUG) """ The STDOUT handler will use the logger default threshold for printing. If the level is set to INFO the STDOUT should only display INFO messages and greater """ if stdout: StdOutHandler = logging.StreamHandler(sys.stdout) StdOutHandler._name = "STDOUT" StdOutHandler.setLevel(level) StdOutHandler.setFormatter(Dispatcher()) self.logger.addHandler(StdOutHandler) if name is None: frame = inspect.stack()[1] name = inspect.getfile(frame[0]).split("/")[-1].split(".py")[0] logFileBase = self.logdir + name + "-" + time.strftime("%Y%m%d_%H%M") """ The Full log will contain every level of output and will be created in any configuration for use when posting the log to the web server. """ fullLogFile = logFileBase + "_FULL.log" self.fullLogFile = fullLogFile FullLogFileHandler = WatchedFileHandler(fullLogFile) FullLogFileHandler.setLevel(level) FullLogFileHandler._name = "LogFile-FULL" FullLogFileHandler.setFormatter(Dispatcher()) self.logger.addHandler(FullLogFileHandler) """ In the case of multiFile = True: Create a FileHandler for each level and attatch the appropriate level name to the file suffix Then set a filter on each handler to return only the appropriate level per file """ if multiFile: # Set up filename variables debugLogFile = logFileBase + "_DEBUG.log" commentLogFile = logFileBase + "_COMMENT.log" infoLogFile = logFileBase + "_INFO.log" warningLogFile = logFileBase + "_WARNING.log" errorLogFile = logFileBase + "_ERROR.log" # Create FileHandler objects DebugFileHandler = WatchedFileHandler(debugLogFile) DebugFileHandler._name = "LogFile-DEBUG" CommentFileHandler = WatchedFileHandler(commentLogFile) CommentFileHandler._name = "LogFile-COMMENT" InfoFileHandler = WatchedFileHandler(infoLogFile) InfoFileHandler._name = "LogFile-INFO" WarningFileHandler = WatchedFileHandler(warningLogFile) WarningFileHandler._name = "LogFile-WARNING" ErrorFileHandler = WatchedFileHandler(errorLogFile) ErrorFileHandler._name = "LogFile-ERROR" # Add filters at corresponding levels DebugFileHandler.addFilter(LogFilter(DEBUG)) CommentFileHandler.addFilter(LogFilter(COMMENT)) InfoFileHandler.addFilter(LogFilter(INFO)) WarningFileHandler.addFilter(LogFilter(WARNING)) ErrorFileHandler.addFilter(LogFilter(ERROR)) # Add format Dispatcher DebugFileHandler.setFormatter(Dispatcher()) CommentFileHandler.setFormatter(Dispatcher()) InfoFileHandler.setFormatter(Dispatcher()) WarningFileHandler.setFormatter(Dispatcher()) ErrorFileHandler.setFormatter(Dispatcher()) # Add handlers to root logger self.logger.addHandler(DebugFileHandler) self.logger.addHandler(CommentFileHandler) self.logger.addHandler(InfoFileHandler) self.logger.addHandler(WarningFileHandler) self.logger.addHandler(ErrorFileHandler)
def logging_init(cls, log_level="INFO", force_reset=False, log_callback=None, log_to_file=None, log_to_syslog=True, log_to_syslog_facility=SysLogHandler.LOG_LOCAL0, log_to_console=True, log_to_file_mode="watched_file", context_filter=None): """ Initialize logging sub system with default settings (console, pre-formatted output) :param log_to_console: if True to to console :type log_to_console: bool :param log_level: The log level to set. Any value in "DEBUG", "INFO", "WARN", "ERROR", "CRITICAL" :type log_level: str :param force_reset: If true, logging system is reset. :type force_reset: bool :param log_to_file: If specified, log to file :type log_to_file: str,None :param log_to_syslog: If specified, log to syslog :type log_to_syslog: bool :param log_to_syslog_facility: Syslog facility. :type log_to_syslog_facility: int :param log_to_file_mode: str "watched_file" for WatchedFileHandler, "time_file" for TimedRotatingFileHandler (or time_file_seconds for unittest) :type log_to_file_mode: str :param log_callback: Callback for unittest :param context_filter: Context filter. If None, pysolbase.ContextFilter.ContextFilter is used. If used instance has an attr "filter", it is added to all handlers and "%(kfilter)s" will be populated by all thread context key/values, using filter method call. Refer to our ContextFilter default implementation for details. :type context_filter: None,object :return Nothing. """ if cls._logging_initialized and not force_reset: return with cls._logging_lock: if cls._logging_initialized and not force_reset: return # Default logging.basicConfig(level=log_level) # Filter if context_filter: c_filter = context_filter else: c_filter = ContextFilter() # Format begin s_f = "%(asctime)s | %(levelname)s | %(module)s@%(funcName)s@%(lineno)d | %(message)s " # Browse if hasattr(c_filter, "filter"): # Push generic field # We expect it to be formatted like our pysolbase.ContextFilter.ContextFilter#filter method. s_f += "|%(kfilter)s" # Format end s_f += "| %(thread)d:%(threadName)s | %(process)d:%(processName)s" # Formatter f = logging.Formatter(s_f) # Console handler c = None if log_to_console: # This can be override by unittest, we use __stdout__ c = logging.StreamHandler(sys.__stdout__) c.setLevel(logging.getLevelName(log_level)) c.setFormatter(f) # File handler to /tmp cf = None if log_to_file: if log_to_file_mode == "watched_file": cf = WatchedFileHandler(log_to_file, encoding="utf-8") cf.setLevel(logging.getLevelName(log_level)) cf.setFormatter(f) elif log_to_file_mode == "time_file": cf = TimedRotatingFileHandler(log_to_file, encoding="utf-8", utc=True, when="D", interval=1, backupCount=7) cf.setLevel(logging.getLevelName(log_level)) cf.setFormatter(f) elif log_to_file_mode == "time_file_seconds": # For unittest only cf = TimedRotatingFileHandler(log_to_file, encoding="utf-8", utc=True, when="S", interval=1, backupCount=7) cf.setLevel(logging.getLevelName(log_level)) cf.setFormatter(f) else: logger.warning("Invalid log_to_file_mode=%s", log_to_file_mode) # Syslog handler syslog = None if log_to_syslog: try: from pysolbase.SysLogger import SysLogger syslog = SysLogger(log_callback=log_callback, facility=log_to_syslog_facility) syslog.setLevel(logging.getLevelName(log_level)) syslog.setFormatter(f) except Exception as e: # This will fail on windows (no attr AF_UNIX) logger.debug("Unable to import SysLogger, e=%s", SolBase.extostr(e)) syslog = False # Initialize root = logging.getLogger() root.setLevel(logging.getLevelName(log_level)) root.handlers = [] if log_to_console: c.addFilter(c_filter) root.addHandler(c) if log_to_file and cf: cf.addFilter(c_filter) root.addHandler(cf) if log_to_syslog and syslog: syslog.addFilter(c_filter) root.addHandler(syslog) # Done cls._logging_initialized = True if force_reset: lifecyclelogger.info( "Logging : initialized from memory, log_level=%s, force_reset=%s", log_level, force_reset) else: lifecyclelogger.debug( "Logging : initialized from memory, log_level=%s, force_reset=%s", log_level, force_reset)
def __init__(self, level=logging.DEBUG, name=None, logdir='./', stdout=True, multiFile=False, post=False, ws='www-qa.coraid.com'): self.logdir = logdir self.ws = ws self.instance = os.environ.get('instance') or '' self.level = level logging.addLevelName(COMMENT, "COMMENT") # Root Logger self.logger = logging.getLogger('otto' + self.instance) self.logger.addHandler(logging.NullHandler()) """ Root Logger Threshold is WARNING by default. We will set the threshold as low as possible """ self.logger.setLevel(DEBUG) """ The STDOUT handler will use the logger default threshold for printing. If the level is set to INFO the STDOUT should only display INFO messages and greater """ if stdout: StdOutHandler = logging.StreamHandler(sys.stdout) StdOutHandler._name = "STDOUT" StdOutHandler.setLevel(level) StdOutHandler.setFormatter(Dispatcher()) self.logger.addHandler(StdOutHandler) if name is None: frame = inspect.stack()[1] name = inspect.getfile(frame[0]).split('/')[-1].split(".py")[0] logFileBase = self.logdir + name + "-" + time.strftime('%Y%m%d_%H%M') """ The Full log will contain every level of output and will be created in any configuration for use when posting the log to the web server. """ fullLogFile = logFileBase + "_FULL.log" self.fullLogFile = fullLogFile FullLogFileHandler = WatchedFileHandler(fullLogFile) FullLogFileHandler.setLevel(level) FullLogFileHandler._name = "LogFile-FULL" FullLogFileHandler.setFormatter(Dispatcher()) self.logger.addHandler(FullLogFileHandler) """ In the case of multiFile = True: Create a FileHandler for each level and attatch the appropriate level name to the file suffix Then set a filter on each handler to return only the appropriate level per file """ if multiFile: # Set up filename variables debugLogFile = logFileBase + "_DEBUG.log" commentLogFile = logFileBase + "_COMMENT.log" infoLogFile = logFileBase + "_INFO.log" warningLogFile = logFileBase + "_WARNING.log" errorLogFile = logFileBase + "_ERROR.log" # Create FileHandler objects DebugFileHandler = WatchedFileHandler(debugLogFile) DebugFileHandler._name = "LogFile-DEBUG" CommentFileHandler = WatchedFileHandler(commentLogFile) CommentFileHandler._name = "LogFile-COMMENT" InfoFileHandler = WatchedFileHandler(infoLogFile) InfoFileHandler._name = "LogFile-INFO" WarningFileHandler = WatchedFileHandler(warningLogFile) WarningFileHandler._name = "LogFile-WARNING" ErrorFileHandler = WatchedFileHandler(errorLogFile) ErrorFileHandler._name = "LogFile-ERROR" # Add filters at corresponding levels DebugFileHandler.addFilter(LogFilter(DEBUG)) CommentFileHandler.addFilter(LogFilter(COMMENT)) InfoFileHandler.addFilter(LogFilter(INFO)) WarningFileHandler.addFilter(LogFilter(WARNING)) ErrorFileHandler.addFilter(LogFilter(ERROR)) # Add format Dispatcher DebugFileHandler.setFormatter(Dispatcher()) CommentFileHandler.setFormatter(Dispatcher()) InfoFileHandler.setFormatter(Dispatcher()) WarningFileHandler.setFormatter(Dispatcher()) ErrorFileHandler.setFormatter(Dispatcher()) # Add handlers to root logger self.logger.addHandler(DebugFileHandler) self.logger.addHandler(CommentFileHandler) self.logger.addHandler(InfoFileHandler) self.logger.addHandler(WarningFileHandler) self.logger.addHandler(ErrorFileHandler)
def setup_logging(self): root_logger = logging.getLogger() if self.args.debug: root_logger.setLevel(logging.DEBUG) else: root_logger.setLevel(logging.INFO) if not self.args.quiet: h = logging.StreamHandler() if self.args.journald: h.setFormatter(JournaldFormatter()) else: h.setFormatter( logging.Formatter( "%(asctime)-15s %(levelname)s %(message)s")) if self.args.debug: h.setLevel(logging.DEBUG) elif self.args.verbose: h.setLevel(logging.INFO) else: h.setLevel(logging.WARN) h.addFilter(DefaultLogFilter(self.args.accesslog)) root_logger.addHandler(h) if self.args.accesslog: from logging.handlers import WatchedFileHandler h = WatchedFileHandler(self.args.accesslog) h.setFormatter(logging.Formatter("%(message)s")) h.setLevel(logging.INFO) class Filter: def filter(self, record): if not getattr(record, "access_log", False): return False return record.levelno < logging.WARN h.addFilter(Filter()) root_logger.addHandler(h) if self.args.perflog: from logging.handlers import WatchedFileHandler h = WatchedFileHandler(self.args.perflog) class PerfFormatter(logging.Formatter): def format(self, record): import json info = getattr(record, "perf", None) return json.dumps(info, sort_keys=True) h.setFormatter(PerfFormatter()) h.setLevel(logging.INFO) class Filter: def filter(self, record): return getattr(record, "perf", None) is not None h.addFilter(Filter()) root_logger.addHandler(h) if self.args.errorlog: from logging.handlers import WatchedFileHandler h = WatchedFileHandler(self.args.errorlog) h.setFormatter( logging.Formatter("%(asctime)-15s %(levelname)s %(message)s")) h.setLevel(logging.WARN) root_logger.addHandler(h) if self.args.syslog: h = logging.SyslogHandler() h.setFormatter( logging.Formatter("%(asctime)-15s %(levelname)s %(message)s")) root_logger.addHandler(h)
log.addHandler(handler_out) class FilterErr(logging.Filter): def filter(self, rec): return is_error(rec.levelno) handler_err = logging.StreamHandler(sys.stderr) handler_err.setFormatter(formatter) if LOG_LEVEL: handler_err.setLevel(LOG_LEVEL) handler_err.addFilter(FilterErr()) log.addHandler(handler_err) if LOG_LEVEL and LOG_DIR: try: os.makedirs(LOG_DIR) except OSError: if not os.path.isdir(LOG_DIR): raise filename = os.path.basename(sys.argv[0]) if filename.endswith('.py'): filename = filename[:-3] filename += '.log' logfile = os.path.join(LOG_DIR, filename) file_handler_err = WatchedFileHandler(logfile) file_handler_err.setFormatter(json_formatter) file_handler_err.setLevel(LOG_LEVEL) file_handler_err.addFilter(FilterErr()) log.addHandler(file_handler_err)