def get_logger(name, to_file=True, to_stdout=False, log_path=STD_LOGPATH, format_str=STD_FORMAT): logger = logging.getLogger(name) if name in LOGGERS: return logger logger.setLevel(logging.INFO) formatter = logging.Formatter(format_str) if to_file: os.system('mkdir -p %s' % log_path) file_path = os.path.join(log_path, '%s.log' % name) handler = RotatingFileHandler(file_path, maxBytes=5e6, backupCount=5) handler.setFormatter(formatter) logger.addHandler(handler) if to_stdout: handler = logging.StreamHandler(sys.stdout) handler.setFormatter(formatter) logger.addHandler(handler) LOGGERS.add(name) return logger
def __init__(self, filename): pwd = os.path.abspath(os.path.dirname(__file__)) directory = os.path.join(pwd, self.folder) if not os.path.exists(directory): os.mkdir(directory) self.file_path = directory + '/' + filename self.log = logging.getLogger(filename) self.log.setLevel(self.level) handler = ConcurrentRotatingFileHandler(self.file_path, 'a', 1024 * 1024 * 100, backupCount=5, encoding='utf-8') # handler.suffix = "%Y-%m-%d" # 设置输出格式 # format_log = "%(asctime)s %(threadName)s %(funcName)s %(filename)s:%(lineno)s %(levelname)s %(message)s" formatter = logging.Formatter( '%(asctime)s [%(processName)s %(threadName)s %(levelname)s %(module)s:%(funcName)s:%(lineno)d] %(message)s' ) # fmt = logging.Formatter(formatter) handler.setFormatter(formatter) self.log.addHandler(handler) # 控制台输出 stream = logging.StreamHandler() stream.setFormatter(formatter) self.log.addHandler(stream)
def set_logger(): log_level = __conf["log"]["level"].upper() log_path = os.path.abspath( os.path.join(os.path.split(__file__)[0], "../logs/threat-detection")) log_size = 4 * 1024 * 1024 log = logging.getLogger("threat_intelligence") if len(log.handlers) == 0: rotate_handler = ConcurrentRotatingFileHandler(log_path, "a", log_size, 5) if log_level == "DEBUG": log.setLevel(logging.DEBUG) elif log_level == "INFO": log.setLevel(logging.INFO) elif log_level == "WARNING": log.setLevel(logging.WARNING) elif log_level == "ERROR": log.setLevel(logging.ERROR) else: raise ValueError, "logLevel should be DEBUG/INFO/WARNING/ERROR." # set logs formatter formatter = logging.Formatter( '%(asctime)s - %(levelname)s - %(message)s') rotate_handler.setFormatter(formatter) # add handler to logger log.addHandler(rotate_handler) return log
def get_logger(ENV, BASE_DIR): # temporary is_email_script = pathlib.Path(sys.argv[0]).name == "email_db_report.py" # use Airbrake in production if(ENV=="production" and not is_email_script): log = airbrake.getLogger() log.setLevel(logging.INFO) else: log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) # print all debug and higher to STDOUT # if the environment is development if(ENV=="development"): stdoutHandler = logging.StreamHandler(sys.stdout) stdoutHandler.setLevel(logging.DEBUG) log.addHandler(stdoutHandler) logfile = os.path.abspath(BASE_DIR + "/logs/CivilServant_" + ENV + ".log") print("Logging to " + BASE_DIR + "/logs/CivilServant_" + ENV + ".log") formatter = logging.Formatter('%(asctime)s - %(name)s({env}) - %(levelname)s - %(message)s'.format(env=ENV)) rotateHandler = ConcurrentRotatingFileHandler(logfile, "a", 32 * 1000 * 1024, 5) rotateHandler.setLevel(logging.DEBUG) rotateHandler.setFormatter(formatter) log.addHandler(rotateHandler) return log
def create_app(): app = Flask(__name__) # 日志模块 rotateHandler = ConcurrentRotatingFileHandler('%s/logs/service.log' % PROJECT_PATH, 'a', 800 * 1024 * 1024, backupCount=10, encoding='utf-8') datefmt_str = '%Y-%m-%d %H:%M:%S' format_str = '%(asctime)s %(levelname)s %(module)s.%(funcName)s Line:%(lineno)d %(message)s' formatter = logging.Formatter(format_str, datefmt_str) rotateHandler.setFormatter(formatter) app.logger.addHandler(rotateHandler) app.logger.setLevel(logging.DEBUG) app.config.from_object(config) config.init_app(app) # 初始化db db.init_app(app) # 初始化cache cache.init_app(app) return app
def setupLogHandlers(fname, formatter=None, **kwargs): """ Create a RotatingFileHandler to be used by a logger, and possibly a GELFHandler. By default the RotatingFileHandler stores 100 MB before starting a new log file and the last 10 log files are kept. The default formatter shows the logging level, current time, the function that created the log entry, and the specified message. :param str fname: path to the filename where logs will be written to :param logging.Formatter formatter: a custom formatter for this logger :param kwargs: custom parameters for the RotatingFileHandler :rtype: tuple """ if formatter is None: formatter = logging.Formatter('%(levelname)s [%(asctime)s] %(funcName)s: %(message)s') opts = {'maxBytes': 100 * 1024 * 1024, 'backupCount': 10, 'debug': False} opts.update(kwargs) handler = RotatingFileHandler(os.path.join(pikaconfig.LOG_DIR, fname), **opts) handler.setFormatter(formatter) handlers = (handler, ) if USE_GELF: gelf_handler = GELFHandler(**USE_GELF) gelf_handler.setLevel(logging.INFO) # Ignore DEBUG messages. handlers += (gelf_handler, ) return handlers
def init_http_logger(config): logger = logging.getLogger(config.LOGGER_HTTP_NAME) del logger.handlers[:] Rthandler = ConcurrentRotatingFileHandler('/tmp/snapperhttp.log', maxBytes=(100*1024), backupCount=1) Rthandler.setFormatter(Formatter(config.LOG_FORMAT)) logger.addHandler(Rthandler) logger.setLevel(logging.DEBUG)
def log_config(f_level=logging.INFO, c_level=logging.CRITICAL, out_path='', filename='info', fix=False): logfile = os.path.join(out_path, filename) + '-' + time.strftime('%Y_%m%d_%H%M%S', time.localtime()) + '.log' \ if not fix else os.path.join(out_path, filename) + '.log' print("2:", logfile) logger = logging.getLogger(logfile) if logger.handlers: logger.removeHandler(logger.handlers) logger.setLevel(f_level) fh = LogHandler(logfile, maxBytes=100 * 1024 * 1024, backupCount=50) fh.setLevel(f_level) ch = logging.StreamHandler() ch.setLevel(c_level) formatter = logging.Formatter( '[%(levelname)s]--%(asctime)s--[%(filename)s %(funcName)s %(lineno)d]: %(message)s' ) fh.setFormatter(formatter) ch.setFormatter(formatter) logger.addHandler(fh) logger.addHandler(ch) return logger, logfile
def __init__(self, path, clevel=logging.DEBUG, Flevel=logging.DEBUG, when='M', backCount=5, fmt='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'): self.logger = logging.getLogger(path) self.logger.setLevel(logging.DEBUG) fmt = logging.Formatter('%(asctime)s | %(message)s', '%Y-%m-%d %H:%M:%S') # fmt = logging.Formatter('%(asctime)s | %(message)s') # Use an absolute path to prevent file rotation trouble. logfile = os.path.abspath(path) # Rotate log after reaching 512K, keep 5 old copies. rh = ConcurrentRotatingFileHandler(logfile, "a", 10 * 1024 * 1024 * 1024, backCount) # th = handlers.TimedRotatingFileHandler(filename=logfile, when=when, backupCount=backCount, encoding='utf-8') rh.setFormatter(fmt) # 设置CMD日志 sh = logging.StreamHandler() sh.setFormatter(fmt) sh.setLevel(clevel) # 设置文件日志 fh = logging.FileHandler(path, encoding='utf-8') fh.setFormatter(fmt) fh.setLevel(Flevel) self.logger.addHandler(sh) # self.logger.addHandler(fh) # self.logger.addHandler(th) self.logger.addHandler(rh)
def add_rotating_file_logger(logger, logfile, log_level=None, format=None, mode="a", maxBytes=10*(1024**2), backupCount=5): """Add a rotating file logger to the logger.""" log_level = log_level or logging.DEBUG format = format or BRIEF_LOG_FORMAT # touch the logfile if not os.path.exists(logfile): try: fo = open(logfile, "w") fo.close() except (ValueError, IOError): return # is the logfile really a file? if not os.path.isfile(logfile): return # check if the logfile is writable if not os.access(logfile, os.W_OK): return handler = RFHandler(logfile, maxBytes=maxBytes, backupCount=backupCount, mode=mode) handler.setFormatter(logging.Formatter(format, datefmt="%Y-%m-%d %H:%M:%S")) handler.setLevel(log_level) logger.addHandler(handler)
def configure_logging(logger, log_filename, log_level=logging.INFO, stderr_level=logging.ERROR, log_dir=LOG_DIR): """Configures logging for given logger using the given filename. :return None. """ # If the logging directory doesn't exist, create it. if not os.path.exists(log_dir): os.makedirs(log_dir) # Determine path to log file. log_path = os.path.join(log_dir, log_filename) # Create an IdentityFilter. identity = get_identifier() identity_filter = IdentityFilter(identity=identity) # Create a log handler and formtter and apply to _log. hdlr = ConcurrentRotatingFileHandler(filename=log_path, maxBytes=1000000, backupCount=5) hdlr.addFilter(identity_filter) formatter = logging.Formatter(LOG_FORMAT) hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.setLevel(log_level) # Attach a stderr handler to the log. stderr_hdlr = logging.StreamHandler(sys.stderr) stderr_hdlr.setLevel(stderr_level) stderr_hdlr.setFormatter(formatter) logger.addHandler(stderr_hdlr)
def startlogging(log, logfile, loglevel=logging.INFO, consolelevel=None): """Start the logging system to store rotational file based log.""" try: from cloghandler import ConcurrentRotatingFileHandler as RFHandler except ImportError: # Next 2 lines are optional: issue a warning to the user from warnings import warn warn("ConcurrentLogHandler package not installed. Using builtin log handler") from logging.handlers import RotatingFileHandler as RFHandler if not consolelevel: consolelevel = loglevel log.setLevel(loglevel) #create file handler and set level to debug fh = RFHandler(filename=logfile, maxBytes=2**20, backupCount=50) fh.setLevel(loglevel) #create console handler and set level to error ch = logging.StreamHandler() ch.setLevel(consolelevel) #create formatter formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") #add formatter to fh fh.setFormatter(formatter) #add formatter to ch ch.setFormatter(formatter) #add fh to logger log.addHandler(fh) #add ch to logger log.addHandler(ch) log.debug("Logging started at level %d", loglevel) return log
def log(message, path=None, level=None, filename=None, log_type=None): if not filename: filename = 'python_logs' if not path: path = './logs/' if not os.path.exists(path): os.makedirs(path) logger = logging.getLogger(filename) if not level: logger.setLevel(logging.INFO) elif level == 'debug': logger.setLevel(logging.DEBUG) # 若logger.handlers列表为空,则添加,否则,直接去写日志,避免重复写入日志 if not logger.handlers: filehandler = ConcurrentRotatingFileHandler(path + filename + '.log') formatter = logging.Formatter( '%(asctime)s|%(levelname)s|%(name)s|%(message)s') filehandler.setFormatter(formatter) logger.addHandler(filehandler) if not log_type: logger.info(message) elif log_type == 'error': logger.error(message) elif log_type == 'warning': logger.warning(message) elif log_type == 'debug': logger.debug(message)
def setup_logging(file_, name_, level=config['LOGGING_LEVEL']): """ Sets up generic logging to file with rotating files on disk :param file_: the __file__ doc of python module that called the logging :param name_: the name of the file that called the logging :param level: the level of the logging DEBUG, INFO, WARN :return: logging instance """ level = getattr(logging, level) logfmt = '%(levelname)s\t%(process)d [%(asctime)s]:\t%(message)s' datefmt = '%m/%d/%Y %H:%M:%S' formatter = logging.Formatter(fmt=logfmt, datefmt=datefmt) logging_instance = logging.getLogger(name_) fn_path = os.path.join(os.path.dirname(file_), PROJ_HOME, 'logs') if not os.path.exists(fn_path): os.makedirs(fn_path) fn = os.path.join(fn_path, '{0}.log'.format(name_)) rfh = ConcurrentRotatingFileHandler(filename=fn, maxBytes=2097152, backupCount=5, mode='a', encoding='UTF-8') # 2MB file rfh.setFormatter(formatter) logging_instance.handlers = [] logging_instance.addHandler(rfh) logging_instance.setLevel(level) return logging_instance
def __init__(self): """ Setup configuration and logging """ try: debug = getattr(settings, 'DEBUG', False) self.mysql_host = getattr(settings, 'MYSQL_HOST') self.mysql_db = getattr(settings, 'MYSQL_DB') self.mysql_user = getattr(settings, 'MYSQL_USER') self.mysql_pass = getattr(settings, 'MYSQL_PASS') self.mysql_ssl = getattr(settings, 'MYSQL_SSL_CA', None) except AttributeError as ex: print('Missing or invalid configuration. Check settings.py') print(str(ex)) if debug: ltype = 'DEBUG' level = logging.DEBUG else: ltype = 'INFO' level = logging.INFO maxsize = 16 * 1024 * 1024 handler = RFHandler(logfile, 'a', maxBytes=maxsize, backupCount=9) form = '%(asctime)s [%(process)d] %(levelname)s: %(message)s' fmat = logging.Formatter(form) fmat.converter = time.gmtime global logg logg = logging.getLogger('ClassExample') logg.propagate = True logg.setLevel(level) handler.setFormatter(fmat) logg.addHandler(handler) form = 'ClassExample %s process starting in %s mode' args = (__version__, ltype) logg.info(form % args)
def build_logger_env(worker_name, log_level=logging.NOTSET): """规范化log输出""" logdir_path = os.path.join(os.path.dirname(__file__), os.pardir, 'log' ) if not os.path.exists(logdir_path): os.makedirs(logdir_path) logger = __loggers.get(worker_name) if logger: return logger logger = logging.getLogger(worker_name) logger.propagate = 0 # 拒绝 父Logger 产生日志 logger.setLevel(log_level) ch = ConcurrentRotatingFileHandler(os.path.join(logdir_path, '%s.log'%worker_name), 'a', 50*1024*1024, 5 ) ch.setLevel(log_level) formatter = logging.Formatter('%(asctime)s %(levelname)s %(module)s.%(funcName)s[%(lineno)d] MSG:%(message)s') ch.setFormatter(formatter) logger.addHandler(ch) __loggers.setdefault(worker_name, logger) return logger
def f_log_concurrent(log_file): LEVELS = { 'debug': logging.DEBUG, 'info': logging.INFO, 'error': logging.ERROR } log = logging.getLogger() level = LEVELS.get(LOGLEVEL, logging.NOTSET) # Use an absolute path to prevent file rotation trouble. logfile = os.path.abspath(log_file) # Rotate log after reaching 1G, keep 60 old copies. rotateHandler = ConcurrentRotatingFileHandler(logfile, "a", 1024 * 1024 * 1024, 60, encoding="utf-8") fm = logging.Formatter( "%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S", ) rotateHandler.setFormatter(fm) log.addHandler(rotateHandler) log.setLevel(level) return log
def get_instance(tag="test"): logging.config.dictConfig(LOGGING) logger = logging.getLogger("clogger") if not os.path.exists(os.path.join(LOG_PATH)): os.makedirs(os.path.join(LOG_PATH)) logfile = os.path.join(LOG_PATH) + '%s.log' % tag fh = RFHandler(logfile, maxBytes=1024 * 1024 * 100, backupCount=10, delay=0.05) formatter = logging.Formatter( '[%(asctime)s - %(levelno)s] - %(message)s') fh.setFormatter(formatter) fh.setLevel(logging.DEBUG) logger.addHandler(fh) # error_logfile = os.path.join(LOG_PATH) + '%s_error.log' % tag # efh = RFHandler(error_logfile, maxBytes=1024 * 1024 * 100, backupCount=10, delay=0.05) # efh.setFormatter(formatter) # efh.setLevel(logging.ERROR) # logger.addHandler(efh) return logger
def setup_logging(file_, name_, level='DEBUG'): """ Sets up generic logging to file with rotating files on disk :param file_: the __file__ doc of python module that called the logging :param name_: the name of the file that called the logging :param level: the level of the logging DEBUG, INFO, WARN :return: logging instance """ level = getattr(logging, level) logfmt = '%(levelname)s\t%(process)d [%(asctime)s]:\t%(message)s' datefmt = '%m/%d/%Y %H:%M:%S' formatter = logging.Formatter(fmt=logfmt, datefmt=datefmt) logging_instance = logging.getLogger(name_) fn_path = os.path.join(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')), 'logs') if not os.path.exists(fn_path): os.makedirs(fn_path) fn = os.path.join(fn_path, '{0}.log'.format(name_)) rfh = ConcurrentRotatingFileHandler(filename=fn, maxBytes=2097152, backupCount=5, mode='a', encoding='UTF-8') # 2MB file rfh.setFormatter(formatter) logging_instance.handlers = [] logging_instance.addHandler(rfh) logging_instance.setLevel(level) return logging_instance
def configure_logging(app): """Configure logging.""" try: from cloghandler import ConcurrentRotatingFileHandler as RotatingFileHandler except ImportError: RotatingFileHandler = logging.handlers.RotatingFileHandler def log_exception(exc_info): """ Override default Flask.log_exception for more verbose logging on exceptions. """ try: oauth_user = request.oauth except AttributeError: oauth_user = None app.logger.error( """ Request: {method} {path} IP: {ip} Agent: {agent_platform} | {agent_browser} {agent_browser_version} Raw Agent: {agent} Oauth2: {oauth_user} """.format( method=request.method, path=request.path, ip=request.remote_addr, agent_platform=request.user_agent.platform, agent_browser=request.user_agent.browser, agent_browser_version=request.user_agent.version, agent=request.user_agent.string, oauth_user=oauth_user, ), exc_info=exc_info ) app.log_exception = log_exception fn = app.config.get('LOG_FILE') if fn is None: fn = os.path.join(app.instance_path, 'logs/adsws.log') if not os.path.exists(os.path.dirname(fn)): os.makedirs(os.path.dirname(fn)) rfh = RotatingFileHandler(fn, maxBytes=1000000, backupCount=10) rfh.setFormatter(logging.Formatter( '%(asctime)s %(levelname)s: %(message)s ' '[in %(pathname)s:%(lineno)d]') ) # NOTE: # Setting the level on just the handler seems to have *no* effect; # setting the level on app.logger seems to have the desired effect. # I do not understand this behavior # rfh.setLevel(app.config.get('LOG_LEVEL', logging.INFO)) app.logger.setLevel((app.config.get('LOG_LEVEL', logging.INFO))) if rfh not in app.logger.handlers: app.logger.addHandler(rfh) app.logger.debug("Logging initialized")
def get_json_log_handler(path, app_name, json_fields): handler = ConcurrentRotatingFileHandler(path, "a", 2 * 1024 * 1024 * 1024, 1) formatter = LogstashFormatter() formatter.defaults['@tags'] = ['collector', app_name] formatter.defaults['@fields'] = json_fields handler.setFormatter(formatter) return handler
def get_json_log_handler(path, app_name, json_fields): handler = ConcurrentRotatingFileHandler( path, "a", 2 * 1024 * 1024 * 1024, 1) formatter = LogstashFormatter() formatter.defaults['@tags'] = ['collector', app_name] formatter.defaults['@fields'] = json_fields handler.setFormatter(formatter) return handler
def setup_logging(name_, level=None, proj_home=None, attach_stdout=False): """ Sets up generic logging to file with rotating files on disk :param: name_: the name of the logfile (not the destination!) :param: level: the level of the logging DEBUG, INFO, WARN :param: proj_home: optional, starting dir in which we'll check for (and create) 'logs' folder and set the logger there :return: logging instance """ if level is None: config = load_config(extra_frames=1, proj_home=proj_home, app_name=name_) level = config.get('LOGGING_LEVEL', 'INFO') level = getattr(logging, level) logfmt = u'%(asctime)s %(msecs)03d %(levelname)-8s [%(process)d:%(threadName)s:%(filename)s:%(lineno)d] %(message)s' datefmt = TIMESTAMP_FMT # formatter = logging.Formatter(fmt=logfmt, datefmt=datefmt) formatter = MultilineMessagesFormatter(fmt=logfmt, datefmt=datefmt) formatter.multiline_marker = u'' formatter.multiline_fmt = u' %(message)s' formatter.converter = time.gmtime logging_instance = logging.getLogger(name_) logging_instance.propagate = False # logging messages are not passed to the handlers of ancestor loggers (i.e., gunicorn) if proj_home: proj_home = os.path.abspath(proj_home) fn_path = os.path.join(proj_home, u'logs') else: fn_path = os.path.join(_get_proj_home(), u'logs') if not os.path.exists(fn_path): os.makedirs(fn_path) fn = os.path.join(fn_path, u'{0}.log'.format(name_.split(u'.log')[0])) rfh = ConcurrentRotatingFileHandler(filename=fn, maxBytes=10485760, backupCount=10, mode=u'a', encoding=u'UTF-8') # 10MB file rfh.setFormatter(formatter) logging_instance.handlers = [] logging_instance.addHandler(rfh) logging_instance.setLevel(level) if attach_stdout: stdout = logging.StreamHandler(sys.stdout) stdout.formatter = get_json_formatter() logging_instance.addHandler(stdout) return logging_instance
def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=0): filename = _relativeToLogPath(filename) ParentHandler.__init__(self, filename, mode, maxBytes, backupCount, encoding, delay)
def __init__(self, app): self.log = logging.getLogger(app) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') rotateHandler = ConcurrentRotatingFileHandler(LOGFILE, "a", 512 * 1024, 5) rotateHandler.setFormatter(formatter) self.log.addHandler(rotateHandler) self.log.setLevel(logging.DEBUG)
def get_logger(): logger = logging.getLogger('job') log_format = '%(asctime)s %(filename)s %(lineno)d %(levelname)s %(message)s' formatter = logging.Formatter(log_format) logfile = os.path.join(collector_agent_path, 'log/job.log') rotate_handler = ConcurrentRotatingFileHandler(logfile, "a", 2000000, 7) rotate_handler.setFormatter(formatter) logger.addHandler(rotate_handler) logger.setLevel(logging.DEBUG) return logger
def __init__(self, filename, mode, encoding=None, delay=0): """ Use the specified filename for streamed logging """ if codecs is None: encoding = None ConcurrentRotatingFileHandler.__init__(self, filename, mode, encoding, delay) self.suffix = "%Y-%m-%d" self.suffix_time = ""
def __init__(self, json=False, stdout=True, name='scrapy-cluster', dir='logs', file='main.log', bytes=25000000, backups=5, level='INFO', format='%(asctime)s [%(name)s] %(levelname)s: %(message)s', propagate=False): ''' @param stdout: Flag to write logs to stdout or file @param json: Flag to write json logs with objects or just the messages @param name: The logger name @param dir: The directory to write logs into @param file: The file name @param bytes: The max file size in bytes @param backups: The number of backups to keep of the file @param level: The logging level string @param format: The log format @param propagate: Allow the log to propagate to other ancestor loggers ''' # set up logger self.logger = logging.getLogger(name) self.logger.setLevel(logging.DEBUG) self.logger.propagate = propagate self.json = json self.log_level = level self.format_string = format if stdout: # set up to std out stream_handler = logging.StreamHandler(sys.stdout) stream_handler.setLevel(logging.DEBUG) formatter = self._get_formatter(json) stream_handler.setFormatter(formatter) self.logger.addHandler(stream_handler) self._check_log_level(level) self.debug("Logging to stdout") else: # set up to file try: # try to make dir os.makedirs(dir) except OSError as exception: if exception.errno != errno.EEXIST: raise file_handler = ConcurrentRotatingFileHandler(dir + '/' + file, maxBytes=bytes, backupCount=backups) file_handler.setLevel(logging.DEBUG) formatter = self._get_formatter(json) file_handler.setFormatter(formatter) self.logger.addHandler(file_handler) self._check_log_level(level) self.debug("Logging to file: {file}".format( file=dir+'/'+file))
def get_logger(): """Setup and tune the application logger""" app_log = logging.getLogger("tornado.application") tornado.log.enable_pretty_logging() # TODO: user dir logfile = path.abspath('node.log') # Rotate log after reaching 512K, keep 5 old copies. rotate_handler = ConcurrentRotatingFileHandler(logfile, 'a', 512 * 1024, 5) formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') rotate_handler.setFormatter(formatter) app_log.addHandler(rotate_handler) return app_log
def __init__(self, app): self.log = logging.getLogger(app) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') rotateHandler = ConcurrentRotatingFileHandler(LOGFILE, "a", 512 * 1024, 5) rotateHandler.setFormatter(formatter) self.log.addHandler(rotateHandler) self.log.setLevel(logging.DEBUG)
def setup_logging(self,level=logging.DEBUG): logfmt = '%(levelname)s\t%(process)d [%(asctime)s]:\t%(message)s' datefmt= '%m/%d/%Y %H:%M:%S' self.formatter = logging.Formatter(fmt=logfmt,datefmt=datefmt) LOGGER = logging.getLogger(self.__class__.__name__) fn = os.path.join(os.path.dirname(__file__),'..','logs','%s.log' % self.__class__.__name__) rfh = ConcurrentRotatingFileHandler(filename=fn,maxBytes=2097152,backupCount=5,mode='a') #2MB file rfh.setFormatter(self.formatter) LOGGER.handlers = [] LOGGER.addHandler(rfh) LOGGER.setLevel(level) return LOGGER
def get_handler(filename, level=None): ext = logging.getLevelName(level).lower() if level is not None else 'log' handler = ConcurrentRotatingFileHandler(''.join([LOG_PATH, filename, '.', ext]), mode='a', maxBytes=MAX_FILE_SIZE, backupCount=5, encoding='utf-8') handler.setFormatter(formatter) if level is not None: handler.setLevel(level) return handler
def __init__(self, filename, backupCount=0, when='D', interval=1, utc=False, mode='a', delay=False, debug=False, suffix='log'): # Call __init__ of ConcurrentRotatingFileHandler ConcurrentRotatingFileHandler.__init__(self, filename, backupCount=backupCount, mode=mode, debug=debug, supress_abs_warn=True, maxBytes=0) self.when = when.upper() self.utc = utc if not suffix: suffix = 'log' if self.when == 'S': self.interval = 1 # one second self.suffix = "%Y-%m-%d_%H-%M-%S." + suffix self.extMatch = (r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(?=." + suffix + ")") elif self.when == 'M': self.interval = 60 # one minute self.suffix = "%Y-%m-%d_%H-%M." + suffix self.extMatch = (r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}(?=." + suffix + ")") elif self.when == 'H': self.interval = 60 * 60 # one hour self.suffix = "%Y-%m-%d_%H." + suffix self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}(?=." + suffix + ")" elif self.when == 'D' or self.when == 'MIDNIGHT': self.interval = 60 * 60 * 24 # one day self.suffix = "%Y-%m-%d." + suffix self.extMatch = r"^\d{4}-\d{2}-\d{2}(?=." + suffix + ")" elif self.when.startswith('W'): self.interval = 60 * 60 * 24 * 7 # one week if len(self.when) != 2: raise ValueError("You must specify a day for weekly rollover " "from 0 to 6 (0 is Monday): %s" % self.when) if self.when[1] < '0' or self.when[1] > '6': raise ValueError("Invalid day specified for weekly rollover: " "%s" % self.when) self.dayOfWeek = int(self.when[1]) self.suffix = "%Y-%m-%d." + suffix self.extMatch = r"^\d{4}-\d{2}-\d{2}(?=." + suffix + ")" else: raise ValueError("Invalid rollover interval specified: %s" % self.when) self.extMatch = re.compile(self.extMatch) self.interval = self.interval * interval # multiply by units requested if os.path.exists(filename): t = os.stat(filename)[ST_MTIME] else: t = int(time.time()) self.rolloverAt = self.computeRollover(t)
def main(): """ 多线程博文下载:采用多线程下载博文,博文根据博客名分文件夹.再根据发布日期分文件夹 python3 download.py 1000 10 启动10个线程下载1000个博文 """ args = sys.argv if len(args) == 2: limit = int(args[1]) thread_num = 8 elif len(args) == 3: limit = int(args[1]) thread_num = int(args[2]) else: limit = 1 thread_num = 1 begin = time.time() # 日志相关初始化 if not os.path.isdir('log'): os.mkdir('log') log_file_name = '%s-%s.log' % (os.path.basename(__file__).replace( '.py', ''), datetime.date.today()) log_full_file_name = os.path.join('log', log_file_name) log = getLogger() rotateHandler = ConcurrentRotatingFileHandler(log_full_file_name, "a", 512 * 1024, 0, 'utf-8') datefmt_str = '%Y-%m-%d %H:%M:%S' format_str = "[%(asctime)s - %(levelname)s - %(filename)s - LINE:%(lineno)d] %(message)s" formatter = Formatter(format_str, datefmt_str) rotateHandler.setFormatter(formatter) log.addHandler(rotateHandler) log.setLevel(INFO) log.info('开始执行: 启动%s个线程下载%s个博文' % (thread_num, limit)) # 实例化线程锁 lock = threading.Lock() # 创建线程池 pool = threadpool.ThreadPool(thread_num) global tqdm_list for x in range(thread_num): tqdm_list[x] = 0 # print(tqdm_list) # print(get_position()) # return False requests_list = [] progress = tqdm(total=limit, desc='total') for x in range(limit): requests_list.append(([x, lock, log, progress, thread_num], None)) requests_res = threadpool.makeRequests(download, requests_list) [pool.putRequest(req) for req in requests_res] pool.wait()
def configuration_logging(**kwargs): log_dir = TORNADO_LOG_SETTINGS['log_file_prefix'] TORNADO_LOG_SETTINGS.update(**kwargs) port = settings.TORNADO_SERVER_SETTINGS['port'] for log_name, logger in [ ('tornado_access_%s.log' % port, tornado.log.access_log), ('tornado_app_%s.log' % port, tornado.log.app_log), ('tornado_gen_%s.log' % port, tornado.log.gen_log) ]: log_file = os.path.join(log_dir, log_name) logger.setLevel( getattr(logging, TORNADO_LOG_SETTINGS['logging'].upper())) if log_file: rotate_mode = TORNADO_LOG_SETTINGS['log_rotate_mode'] if rotate_mode == 'size': channel = RFHandler( log_file, "a", TORNADO_LOG_SETTINGS['log_file_max_size'], TORNADO_LOG_SETTINGS['log_file_num_backups']) # # channel = logging.handlers.RotatingFileHandler( # filename=log_file, # maxBytes=TORNADO_LOG_SETTINGS['log_file_max_size'], # backupCount=TORNADO_LOG_SETTINGS['log_file_num_backups'] # ) elif rotate_mode == 'time': channel = logging.handlers.TimedRotatingFileHandler( filename=log_file, when=TORNADO_LOG_SETTINGS['log_rotate_when'], interval=TORNADO_LOG_SETTINGS['log_rotate_interval'], backupCount=TORNADO_LOG_SETTINGS['log_file_num_backups']) else: error_message = 'The value of log_rotate_mode option should be ' + \ '"size" or "time", not "%s".' % rotate_mode raise ValueError(error_message) channel.setFormatter( tornado.log.LogFormatter( fmt=TORNADO_LOG_SETTINGS['log_fmt'], datefmt=TORNADO_LOG_SETTINGS['log_datefmt'], color=False)) logger.addHandler(channel) if (TORNADO_LOG_SETTINGS['log_to_stderr'] or (TORNADO_LOG_SETTINGS['log_to_stderr'] is None and not logger.handlers)): # Set up color if we are in a tty and curses is installed channel = logging.StreamHandler() channel.setFormatter( tornado.log.LogFormatter( fmt=TORNADO_LOG_SETTINGS['log_fmt'], datefmt=TORNADO_LOG_SETTINGS['log_datefmt'])) logger.addHandler(channel)
def __init__(self, name): logging.Logger.__init__(self, name, self.level) if self.filename is not None: # Rotate log after reaching sizelimit, keep 25 old copies. handler = ConcurrentRotatingFileHandler(self.filename, "a", self.sizelimit, 25) else: handler = logging.StreamHandler() if self.level == logging.DEBUG: handler.setFormatter(self.debug_formatter()) else: handler.setFormatter(self.info_formatter()) self.addHandler(handler)
def init_logger(config): # Remove exist handlers logger = logging.getLogger(config.LOGGER_NAME) del logger.handlers[:] Rthandler = ConcurrentRotatingFileHandler('/tmp/snapperapp.log', maxBytes=config.LOG_FILE_SIZE, backupCount=config.LOG_BKUP_FILES) Rthandler.setFormatter(Formatter(config.LOG_FORMAT)) logger.addHandler(Rthandler) logger.setLevel(config.LOG_LEVEL) if config.LOG_CONSOLE == True: console = logging.StreamHandler() console.setFormatter(Formatter(config.LOG_FORMAT)) logger.addHandler(console) logger.setLevel(config.LOG_CONSOLE_LEVEL)
def emit(self, record): """ Emit a record. Always check time """ try: if self.check_baseFilename(record): self.build_baseFilename() ConcurrentRotatingFileHandler.emit(self, record) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record)
def init_logger(cls, port): formatter = LogFormatter(fmt=cls.__fmt.format(port=port), datefmt="", color=False) access_log_handler = ConcurrentRotatingFileHandler( filename=os.path.join(ServerConfig["log_dir"], "access.log")) access_log_handler.setFormatter(formatter) access_log.addHandler(access_log_handler) server_log_handler = ConcurrentRotatingFileHandler( filename=os.path.join(ServerConfig['log_dir'], 'server.log'), maxBytes=128 * 1024 * 1024, backupCount=5, encoding='utf8') server_log_handler.setFormatter(formatter) gen_log.addHandler(server_log_handler) app_log.addHandler(server_log_handler) access_log.setLevel(logging.INFO) gen_log.setLevel(getattr(logging, ServerConfig['log_level'].upper())) app_log.setLevel(getattr(logging, ServerConfig['log_level'].upper())) access_log.propagate = app_log.propagate = gen_log.propagate = False return
def setup_logging(name_, level=None, proj_home=None): """ Sets up generic logging to file with rotating files on disk :param: name_: the name of the logfile (not the destination!) :param: level: the level of the logging DEBUG, INFO, WARN :param: proj_home: optional, starting dir in which we'll check for (and create) 'logs' folder and set the logger there :return: logging instance """ if level is None: config = load_config(extra_frames=1, proj_home=proj_home) level = config.get('LOGGING_LEVEL', 'INFO') level = getattr(logging, level) logfmt = u'%(asctime)s,%(msecs)03d %(levelname)-8s [%(process)d:%(threadName)s:%(filename)s:%(lineno)d] %(message)s' datefmt = u'%Y-%m-%d %H:%M:%S' #formatter = logging.Formatter(fmt=logfmt, datefmt=datefmt) formatter = MultilineMessagesFormatter(fmt=logfmt, datefmt=datefmt) formatter.multiline_marker = '' formatter.multiline_fmt = ' %(message)s' formatter.converter = time.gmtime logging_instance = logging.getLogger(name_) if proj_home: proj_home = os.path.abspath(proj_home) fn_path = os.path.join(proj_home, 'logs') else: fn_path = os.path.join(_get_proj_home(), 'logs') if not os.path.exists(fn_path): os.makedirs(fn_path) fn = os.path.join(fn_path, '{0}.log'.format(name_.split('.log')[0])) rfh = ConcurrentRotatingFileHandler(filename=fn, maxBytes=10485760, backupCount=10, mode='a', encoding='UTF-8') # 10MB file rfh.setFormatter(formatter) logging_instance.handlers = [] logging_instance.addHandler(rfh) logging_instance.setLevel(level) return logging_instance
def initialize_logger(self): # Invoke logging with a concurrent logging module since many of these # processes will likely be writing to scan.log at the same time self.dbg_h = logging.getLogger('dbg_log') dbglog = '%s/%s' % (self.log_path, 'dbg.log') dbg_rotateHandler = ConcurrentRotatingFileHandler(dbglog, "a") self.dbg_h.addHandler(dbg_rotateHandler) self.dbg_h.setLevel(logging.ERROR) self.scan_h = logging.getLogger('scan_log') scanlog = '%s/%s' % (self.log_path, 'scan.log') scan_rotateHandler = ConcurrentRotatingFileHandler(scanlog, "a") self.scan_h.addHandler(scan_rotateHandler) self.scan_h.setLevel(logging.INFO)
def __cmd__(self, lock, command): log = getLogger() # buid log path sefl.__log_id__ = self.__runner_requestor__ + '_' + time.strftime( "%Y_%H_%M_%S") + '.log' logfile = os.path.join(log_path, self.__log_id__) # Rotate log after reaching 10M, keep 5 old copies. rotateHandler = ConcurrentRotatingFileHandler(logfile, "a", 1024 * 1024 * 10, 5) log.addHandler(rotateHandler) p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, errout) = p.communicate() if stdout: lock.acquire() self.__return_content__ = +command + ':' + stdout lock.release() log.setLevel(INFO) log.info( time.strftime("%b %d %Y %H:%M:%S: ") + command + ':' + stdout) if errout: lock.acquire() self.__return_content__ = +command + ':' + errout lock.release() log.setLevel(ERROR) log.info( time.strftime("%b %d %Y %H:%M:%S: ") + command + ':' + errout)
def set_app_log(app): """ 设置flask自带的log :param app: :return: """ l_format = logging.Formatter( "%(asctime)s [%(levelname)s] [%(filename)s %(funcName)s %(lineno)d]: %(message)s " ) r_handler = LogHandler("app.log", maxBytes=20480000, backupCount=10, encoding='UTF-8') r_handler.setLevel(logging.INFO) r_handler.setFormatter(l_format) app.logger.addHandler(r_handler)
def log_add(): logfile = os.path.abspath("mylogfile.log") log = getLogger() rotate_handler = ConcurrentRotatingFileHandler(logfile, "a", 1024*1024, 5) log.addHandler(rotate_handler) log.setLevel(INFO) log.info("Here is a very exciting log message for you.")
def __init__(self, logfile, log_size_limit, log_rotate_num, log_level): self.logger = logging.getLogger() try: self.rotateHandler = ConcurrentRotatingFileHandler( logfile, "a", log_size_limit, log_rotate_num) except Exception, e: print 'INTERNAL_ERR'
def initializeLogging(self,**kwargs): logfmt = '%(levelname)s\t%(process)d [%(asctime)s]:\t%(message)s' datefmt= '%m/%d/%Y %H:%M:%S' formatter = logging.Formatter(fmt=logfmt,datefmt=datefmt) LOGGER = logging.getLogger('PipelineMongoConnection') if not LOGGER.handlers: default_fn = os.path.join(os.path.dirname(__file__),'..','logs','PipelineMongoConnection.log') fn = kwargs.get('logfile',default_fn) rfh = ConcurrentRotatingFileHandler(filename=fn,maxBytes=2097152,backupCount=10,mode='a') #2MB file rfh.setFormatter(formatter) ch = logging.StreamHandler() #console handler ch.setFormatter(formatter) #LOGGER.addHandler(ch) LOGGER.addHandler(rfh) LOGGER.setLevel(logging.DEBUG) self.logger = LOGGER
def __init__(self): logger = logging.getLogger() logger.propagate = False formatter = logging.Formatter('%(levelname)s-%(asctime)s-%(funcName)s-%(message)s') filehandler = RFHandler("/var/log/fanclley.log",'a',1*1024*1024, 10000) filehandler.setFormatter(formatter) filehandler.suffix = "%Y%m%d-%H%M.log" logger.setLevel(logging.INFO) logger.addHandler(filehandler) terminal_handler = logging.StreamHandler() terminal_handler.setFormatter(formatter) terminal_handler.setLevel(logging.DEBUG) logger.addHandler(terminal_handler) self._logger = logger
def init_logging(stdout_enabled=True): root_logger = logging.getLogger() root_logger.setLevel(logging.getLevelName(config.get('log_level', 'INFO'))) # root_logger.setLevel(logging.WARN) logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.ERROR) logging.getLogger('boto').setLevel(logging.ERROR) logging.getLogger('urllib3.connectionpool').setLevel(logging.WARN) log_formatter = logging.Formatter( fmt='%(asctime)s | ' + ECID + ' | %(name)s | %(processName)s | %(levelname)s | %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') stdout_logger = logging.StreamHandler(sys.stdout) stdout_logger.setFormatter(log_formatter) root_logger.addHandler(stdout_logger) if stdout_enabled: stdout_logger.setLevel(logging.getLevelName(config.get('log_level', 'INFO'))) # base log file log_file_name = '%s/migrator.log' % config.get('log_dir') # ConcurrentRotatingFileHandler rotating_file = ConcurrentRotatingFileHandler(filename=log_file_name, mode='a', maxBytes=404857600, backupCount=0) rotating_file.setFormatter(log_formatter) rotating_file.setLevel(logging.INFO) root_logger.addHandler(rotating_file) error_log_file_name = '%s/migrator_errors.log' % config.get('log_dir') error_rotating_file = ConcurrentRotatingFileHandler(filename=error_log_file_name, mode='a', maxBytes=404857600, backupCount=0) error_rotating_file.setFormatter(log_formatter) error_rotating_file.setLevel(logging.ERROR) root_logger.addHandler(error_rotating_file)
def setup_logger(log_filename): logger = logging.getLogger() numeric_level = getattr(logging, settings.LOGGING['level'].upper(), None) if not isinstance(numeric_level, int): raise ValueError('incorrect log level %s in settings' % settings.LOGGING['level']) logger.setLevel(numeric_level) formatter = logging.Formatter('[%(asctime)s - %(name)s - %(levelname)s] %(message)s') # console handler ch = logging.StreamHandler() ch.setFormatter(formatter) logger.addHandler(ch) # # file handler log_filepath = os.path.join(settings.LOGGING['dirname'], log_filename) rotate_handler = RFHandler(log_filepath, "a", settings.LOGGING['max_size'] * 1024 * 1024, settings.LOGGING['backup_num']) rotate_handler.setFormatter(formatter) logger.addHandler(rotate_handler)
def configure_logger(logger, log_level, docker_id=None, log_format=LOG_FORMAT, log_dir=LOG_DIR): """ Configures logging to the file 'calico.log' in the specified log directory If the logs are not coming from calico_kubernetes.py, format the log to include the filename of origin Additionally configures a stderr handler which logs INFO and above to stderr. :param logger: logger object to configure :param log_level: level at which logger starts logging. :param log_format: Indicates which logging scheme to use. :param log_dir: Directory where calico.log lives. If None set to default :return: """ if not os.path.exists(log_dir): os.makedirs(log_dir) formatter = logging.Formatter(log_format) docker_filter = IdentityFilter(identity=docker_id) file_hdlr = ConcurrentRotatingFileHandler(filename=log_dir+'calico.log', maxBytes=1000000, backupCount=5) file_hdlr.setFormatter(formatter) if docker_id: file_hdlr.addFilter(docker_filter) # Add file handler and set log level. logger.addHandler(file_hdlr) logger.setLevel(log_level) # Create a stderr handler and apply it to the logger. # This only logs INFO and above to stderr. stderr_hdlr = logging.StreamHandler(sys.stderr) stderr_hdlr.setLevel(logging.INFO) stderr_hdlr.setFormatter(formatter) logger.addHandler(stderr_hdlr)
def __init__(self, module=''): today_datetime = dt.now() today_date = dt.date(today_datetime) string_date = str(today_date) if module == '': file_name = LOGGER_FILE + string_date else: file_name = LOGGER_FILE + module + '-' + string_date logger = logging.getLogger(file_name) # log_namespace can be replaced with your namespace logger.setLevel(logging.DEBUG) if not logger.handlers: file_name = os.path.join(LOGGING_DIR, '%s.log' % file_name) # usually I keep the LOGGING_DIR defined in some global settings file handler = ConcurrentRotatingFileHandler(file_name) formatter = logging.Formatter('%(asctime)s %(levelname)s:%(module)s:%(message)s') handler.setFormatter(formatter) handler.setLevel(logging.DEBUG) logger.addHandler(handler) self._logger = logger
def __init__(self, url, dirname, secret): log_level = logging.DEBUG log_dir = os.getcwd() log_max_size = 78 * 1024 * 1024 log_max_rotate = 9 LOG_LONG_FORMAT = '[%(module)s][%(funcName)s][%(lineno)d][%(levelname)s][%(message)s]' LOG_RECORD_TIME = '[%(asctime)s]' LOG_ROOT_NAME = 'server-node.log' log = logging.getLogger() log_format = LOG_RECORD_TIME + \ '[' + time.strftime("%Z", time.localtime()) + ']' + \ '[%(process)d]' + \ LOG_LONG_FORMAT log_name = LOG_ROOT_NAME + '.' + str(socket.gethostname()) log_file = os.path.join(log_dir, log_name) try: rotate_handler = ConcurrentRotatingFileHandler(log_file, mode="a", maxBytes=log_max_size, backupCount=log_max_rotate) log.addHandler(rotate_handler) rotate_handler.setFormatter(formatter) except: logging.basicConfig(filename=log_file, level=logging.DEBUG, format='[%(asctime)s ][%(levelname)s][%(message)s]', datefmt='%m/%d/%Y %I:%M:%S %p') pass formatter = logging.Formatter(log_format) log.setLevel(log_level) logging.info('Logging level has been set to DEBUG mode') logging.info('New node started with url <{}> serving directory <{}> with secret<{}>'.format(url, dirname, secret)) #log_file = "server-node.log" #logging.basicConfig(filename=log_file, level=logging.DEBUG, format='[%(asctime)s ][%(levelname)s][%(message)s]', datefmt='%m/%d/%Y %I:%M:%S %p') self.url = url self.dirname = dirname self.secret = secret self.known = set()
def configure_logger(logger, log_level, log_format=LOG_FORMAT, log_to_stdout=True, log_dir=LOG_DIR): """ Configures logging to the file 'calico.log' in the specified log directory If the logs are not coming from calico_kubernetes.py, format the log to include the filename of origin Additionally configures a stdout handler which logs INFO and above to stdout. :param logger: logger object to configure :param log_level: level at which logger starts logging. :param log_format: Indicates which logging scheme to use. :param log_to_stdout: If True, configure the stdout stream handler. :param log_dir: Directory where calico.log lives. If None set to default :return: """ if not os.path.exists(log_dir): os.makedirs(log_dir) formatter = logging.Formatter(log_format) file_hdlr = ConcurrentRotatingFileHandler(filename=log_dir+'calico.log', maxBytes=1000000, backupCount=5) file_hdlr.setFormatter(formatter) logger.addHandler(file_hdlr) logger.setLevel(log_level) # Create an stdout handler and apply it to the logger if log_to_stdout: stdout_hdlr = logging.StreamHandler(sys.stdout) stdout_hdlr.setLevel(log_level) stdout_hdlr.setFormatter(formatter) logger.addHandler(stdout_hdlr)
try: from ads.ADSCachedExports import ADSRecords, init_lookers_cache from lib import conversions except ImportError: print "Unable to import ads.ADSExports.ADSRecords!" print "We will be unable to query ADS-classic for records!" INIT_LOOKERS_CACHE = init_lookers_cache logfmt = "%(levelname)s\t%(process)d [%(asctime)s]:\t%(message)s" datefmt = "%m/%d/%Y %H:%M:%S" formatter = logging.Formatter(fmt=logfmt, datefmt=datefmt) logger = logging.getLogger("ReadRecords") if not logger.handlers: fn = os.path.join(os.path.dirname(__file__), "..", "logs", "ReadRecords.log") rfh = ConcurrentRotatingFileHandler(filename=fn, maxBytes=2097152, backupCount=10, mode="a") # 2MB file rfh.setFormatter(formatter) ch = logging.StreamHandler() # console handler ch.setFormatter(formatter) # logger.addHandler(ch) logger.addHandler(rfh) logger.setLevel(logging.INFO) def canonicalize_records(records, targets=None): """ Takes a dict of {bibcode:fingerprint} and resolves each bibcode to its canonical. Finds all alternates associated with that bibcode and constructs the full JSON_fingerprint from all of these associated records
def get_log_file_handler(path): handler = ConcurrentRotatingFileHandler( path, "a", 2 * 1024 * 1024 * 1024, 1) handler.setFormatter(logging.Formatter( "%(asctime)s [%(levelname)s] -> %(message)s")) return handler
def init(): logger['root_logger'] = logging.getLogger() logfile = os.path.abspath('debug.log') debug_handler = ConcurrentRotatingFileHandler(logfile, 'a', 10485760, 5) debug_handler.setLevel(logging.DEBUG) debug_handler.setFormatter(default_formatter) logger['root_logger'].addHandler(debug_handler) logfile = os.path.abspath('info.log') info_handler = ConcurrentRotatingFileHandler(logfile, 'a', 10485760, 5) info_handler.setLevel(logging.INFO) info_handler.setFormatter(default_formatter) logger['root_logger'].addHandler(info_handler) logfile = os.path.abspath('error.log') error_handler = ConcurrentRotatingFileHandler(logfile, 'a', 10485760, 5) error_handler.setLevel(logging.ERROR) error_handler.setFormatter(default_formatter) logger['root_logger'].addHandler(error_handler)
session = requests.Session() if self.auth_token: _log.debug("Using Auth Token: %s", self.auth_token) session.headers.update({'Authorization': 'Bearer ' + self.auth_token}) response = session.get(path, verify=False) return json.loads(response.text) if __name__ == '__main__': # Configure logging. log_file = "/var/log/calico/kubernetes/policy/agent.log" if not os.path.exists(os.path.dirname(log_file)): os.makedirs(os.path.dirname(log_file)) formatter = logging.Formatter(LOG_FORMAT) file_hdlr = ConcurrentRotatingFileHandler(filename=log_file, maxBytes=1000000, backupCount=5) file_hdlr.setFormatter(formatter) _log.addHandler(file_hdlr) _log.setLevel(LOG_LEVEL) # Log to stderr as well. stdout_hdlr = logging.StreamHandler(sys.stderr) stdout_hdlr.setLevel(LOG_LEVEL) stdout_hdlr.setFormatter(formatter) _log.addHandler(stdout_hdlr) try: PolicyAgent().run() except Exception: # Log the exception
from lustre import lustreapi import subprocess from celery import Celery from celery.decorators import periodic_task from billiard import current_process import memcache import shutil clib = ctypes.CDLL('libc.so.6', use_errno=True) logger = logging.getLogger(__name__) rotateHandler = ConcurrentRotatingFileHandler("/var/log/cmover_del.log", "a", 128*1024*1024) formatter = logging.Formatter('%(asctime)s - %(levelname)s [%(filename)s:%(lineno)s - %(funcName)20s()] - %(message)s') rotateHandler.setFormatter(formatter) logger.addHandler(rotateHandler) REPORT_INTERVAL = 30 # seconds with open('rabbitmq/rabbitmq.conf','r') as f: rabbitmq_server = f.read().rstrip() with open('rabbitmq/rabbitmq_%s.conf'%USERNAME,'r') as f: rabbitmq_password = f.read().rstrip() app = Celery(USERNAME, broker='amqp://%s:%s@%s/%s'%(USERNAME, rabbitmq_password, rabbitmq_server, USERNAME)) app.config_from_object(settings)
def getLogger(logname='root'): logger = logging.getLogger(logname) logger.setLevel(logging.DEBUG) DIR = '../log/' if DIR: # not none subprocess.check_call(['mkdir', '-p', DIR]) #================================ # File Handler #================================ LOG_FILENAME = os.path.abspath(DIR + logname + '.err') handler = ConcurrentRotatingFileHandler(LOG_FILENAME, "a", 200*1024*1024, 5) handler.setLevel(logging.WARN) formatter = logging.Formatter("%(asctime)s\t%(name)s-%(process)s-%(threadName)s\t%(levelname)s\t%(message)s") handler.setFormatter(formatter) logger.addHandler(handler) #================================ # Standard Output Handler: INFO ONLY #================================ # handler = logging.StreamHandler(sys.stdout) LOG_FILENAME = os.path.abspath(DIR + logname + '.info') handler = ConcurrentRotatingFileHandler(LOG_FILENAME, "a", 200*1024*1024, 5) handler.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s\t%(name)s-%(process)s-%(threadName)s\t%(message)s") handler.setFormatter(formatter) handler.addFilter(LevelFilter(logging.INFO)) logger.addHandler(handler) return logger
#!/usr/bin/env python #-*- coding: utf-8 -*- import logging import logging.handlers from cloghandler import ConcurrentRotatingFileHandler import time from random import choice logger = logging.getLogger() #handler = logging.handlers.RotatingFileHandler("logs/output.log","a",1024*1024*100,50) handler = ConcurrentRotatingFileHandler("logs/output.log", "a", 1024*1024*100, 50) formatter = logging.Formatter('%(asctime)s - %(levelname)s: %(message)s') handler.setFormatter(formatter) logger.addHandler(handler) logger.setLevel(logging.DEBUG) ip_array = ['172.16.0.111','192.168.2.9','10.8.0.3','172.15.3.22'] protocol_array = ['tcp','http','udp','icmp'] method_arry = ['PUT','GET','POST'] request_array = ['/','/hello/test','/happy/go','/sos/test'] header_arry = ['NULL','main:default:WAP:LABEL_CX-G_NORMAL_IP',\ 'apikey=7f8c4da3ce9849ffb2134f075201c45a&language=zh-CN&details=true',\ 'tcpmux: peer(218.8.127.165:9800) has 40 connections',\ 'iid=9529137899&device_id=35501623937&ac=4g&channel=store_aliyunos&aid=32&app_name=video_article&version_code=588&version_name=5.8.8&device_platform=android&user_version=1.1.8&ab_version=126489%2C125853%2C124728&ab_feature=z1&ssmix=a&device_type=Bird+D10&device_brand=BIRD&language=zh&os_api=22&os_version=5.1&uuid=862807030595172&openudid=32f5f18b092ba71e&manifest_version_code=118&resolution=720*1280&dpi=320&update_version_code=5881&_rticket=1496198125276'] while 1: num = choice(range(1,20000)) for i in range(num): message = choice(ip_array) + ' ' + choice(protocol_array) + ' ' +\ choice(method_arry) + ' ' + choice(request_array) + ' ' + choice(header_arry) logger.info(message)