def setup_logging() -> Generator[None, None, None]: logging.getLogger("discord").setLevel(logging.INFO) logging.getLogger("discord.http").setLevel(logging.WARNING) log = logging.getLogger() try: log.setLevel(logging.INFO) handler = RotatingFileHandler( filename="fizzbotz.log", mode="w", maxBytes=5 * 1024 * 1024, encoding="utf-8", ) dt_fmt = "%Y-%m-%d %H:%M:%S" fmt = logging.Formatter( "[{asctime}] [{levelname:<7}] {name}: {message}", dt_fmt, style="{") handler.setFormatter(fmt) log.addHandler(handler) yield finally: handlers = log.handlers[:] for handler in handlers: handler.close() log.removeHandler(handler)
def dbl(self,logmsg): try: #locate path for debug log in prefs file logfilePath=os.path.join(self.rootFolderPath,'errors') fileName = 'debugLog' logSize = 5000000 logCount = 5 #if path does not exist, create it if not os.path.exists(logfilePath): os.makedirs(logfilePath) if os.path.exists(logfilePath): logHandler = RotatingFileHandler(logfilePath + "/" + fileName,"a", logSize, logCount) logFormatter = logging.Formatter("%(asctime)s:%(message)s") logHandler.setFormatter(logFormatter) logger = logging.getLogger(__name__) logger.disabled = False logger.addHandler(logHandler) logger.setLevel(logging.DEBUG) logger.debug(logmsg) logHandler.flush() logHandler.close() logger.removeHandler(logHandler) except Exception: #if we can't write to the log for any reason, eat the error and continue. pass
def dbl(self,logmsg): try: #locate path for debug log in prefs file logfilePath=os.path.join(self.rootpath,'debugLogs') fileName = 'debugLog' logSize = 5000000 logCount = 5 #if path does not exist, create it if not os.path.exists(logfilePath): os.makedirs(logfilePath) if os.path.exists(logfilePath): env = envUtils.getEnvironment() self.backupReport += "<tr><td>%s - %s</td></tr>" % (env.localizeUTCDate(env.formatUTCDate()), logmsg) logHandler = RotatingFileHandler(logfilePath + "/" + fileName,"a", logSize, logCount) logFormatter = logging.Formatter("%(asctime)s:%(message)s") logHandler.setFormatter(logFormatter) logger = logging.getLogger(__name__) logger.disabled = False logger.addHandler(logHandler) logger.setLevel(logging.DEBUG) logger.debug(logmsg) logHandler.flush() logHandler.close() logger.removeHandler(logHandler) except Exception: #if we can't write to the log for any reason, eat the error and continue. pass
def instance(cls): if cls.logger is None: #TODO: config to limit logs by days path = "/temporary/log" if not os.path.exists(path): os.makedirs(path) filename = '%s/dubhe.log' % path formatter = logging.Formatter( '%(asctime)s | %(levelname)-8s | %(filename)s:%(lineno)s: %(message)s' ) # 控制台输出 streamHandler = logging.StreamHandler() streamHandler.setFormatter(formatter) # 滚动文件输出 log_file_handler = RotatingFileHandler(filename=filename, mode='a', maxBytes=10 * 1024 * 1024, backupCount=14, encoding=None, delay=0) log_file_handler.setFormatter(formatter) cls.logger = logging.getLogger() cls.logger.setLevel(logging.INFO) cls.logger.addHandler(streamHandler) cls.logger.addHandler(log_file_handler) streamHandler.close() log_file_handler.close() # 过滤kafka INFO级别日志 kafka_logger = logging.getLogger('kafka') kafka_logger.setLevel(logging.WARNING) return cls.logger
def setup_logging(): try: # __enter__ if os.getenv('LOGGING').lower() == 'true': if os.getenv('DEBUG').lower() == 'true': logging.getLogger('discord').setLevel(logging.DEBUG) logging.getLogger('discord.http').setLevel(logging.DEBUG) else: logging.getLogger('discord').setLevel(logging.INFO) logging.getLogger('discord.http').setLevel(logging.WARNING) log = logging.getLogger() log.setLevel(logging.INFO) handler = RotatingFileHandler( filename='logs/log.log', encoding='utf-8', mode='w', maxBytes=32 * 1024 * 1024, backupCount=5) formatter = logging.Formatter( '[{asctime}] [{levelname:<7}] {name}: {message}', '%Y-%m-%d %H:%M:%S', style='{') handler.setFormatter(formatter) log.addHandler(handler) if os.getenv('DEBUG').lower() == 'true': log.info('Logger successfully set up in debug mode') else: log.info('Logger successfully set up in normal mode') yield finally: # __exit__ if os.getenv('LOGGING').lower() == 'true': handlers = log.handlers[:] for handler in handlers: handler.close() log.removeHandler(handler)
def set_logging(): try: logging.getLogger('discord').setLevel(logging.INFO) logging.getLogger('discord.http').setLevel(logging.WARNING) logging.getLogger('discord.state').setLevel(logging.DEBUG) log = logging.getLogger() log.setLevel(logging.DEBUG) date_format = '%Y-%m-%d %H:%M:%S' handler = RotatingFileHandler(filename=config.LOG_FILENAME, encoding='utf-8', mode='a') fmt = logging.Formatter('[{asctime}] [{levelname}] {name}: {message}', date_format, style='{') handler.setFormatter(fmt) log.addHandler(handler) yield finally: for handler in log.handlers[:]: handler.close() log.removeHandler(handler)
def __init__(self): """ 初始化 """ self.logger = logging.getLogger() self.logger.setLevel(logging.INFO) today = time.strftime("%Y-%m-%d", time.localtime(time.time())) if not self.logger.handlers or self.logger.handlers[ 0].baseFilename.find(today) < 0: self.logger.handlers = [] path = os.path.abspath( os.path.join(os.path.dirname((os.path.abspath(__file__))), "logfile")) if not os.path.exists(path): os.makedirs(path) log_filename = os.path.join(path, '{time}.log'.format(time=today)) file_handler = RotatingFileHandler(log_filename, maxBytes=10240, encoding='utf-8') file_handler.setFormatter( logging.Formatter( '%(asctime)s %(filename)s %(name)s line:%(lineno)d %(levelname)s %(message)s' )) self.logger.addHandler(file_handler) file_handler.close()
def __printconsole(self, level, message): # 创建一个logger logger = logging.getLogger() logger.setLevel(logging.DEBUG) # 创建一个handler,用于输出到控制台 ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) # 创建一个handler,用于写入日志文件,设置日志大小和备份数 # fh = logging.FileHandler(self.logname, mode='a', encoding='utf-8') fh = RotatingFileHandler(self.logname, mode='a', encoding='utf-8', maxBytes=1024 * 1024, backupCount=50) fh.setLevel(logging.DEBUG) # 再创建一个handler,用于输出到控制台 ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) # 定义handler的输出格式 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) ch.setFormatter(formatter) # 给logger添加handler logger.addHandler(fh) logger.addHandler(ch) # 记录一条日志 if level == 'info': logger.info(message) elif level == 'debug': logger.debug(message) elif level == 'warning': logger.warning(message) elif level == 'error': logger.error(message) # 移除handler并关闭打开的文件 logger.removeHandler(ch) logger.removeHandler(fh) fh.close()
class MultiProcessingLog(logging.Handler): def __init__(self, filename, mode="a", maxBytes=0, backupCount=0, encoding=None, delay=0): logging.Handler.__init__(self) # In case our call to RotatingFileHandler blows up we first set # the _handler to None self._handler = None self._handler = RotatingFileHandler(filename, mode, maxBytes, backupCount, encoding, delay) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): if self._handler is not None: self._handler.close() logging.Handler.close(self)
def __init__(self, logger=None, level='info'): if not os.path.exists(LOG_DIR): os.mkdir(LOG_DIR) # 先获取记录器: self.logger = logging.getLogger(logger) # 设置日志等级 self.logger.setLevel(self.levels[level]) # 设置日志输出格式 formatter = logging.Formatter('[%(asctime)s] [%(pathname)s:%(lineno)d] - %(levelname)s: %(message)s') # 设置日志文件及回滚 frHandler = RotatingFileHandler(LOG_FILE, maxBytes=20 * 1024 * 1024, backupCount=30, encoding='utf-8') frHandler.setLevel(self.levels[level]) frHandler.setFormatter(formatter) # 设置console输出 console = logging.StreamHandler() console.setLevel(self.levels[level]) console.setFormatter(formatter) # 添加以上两个handler self.logger.addHandler(frHandler) self.logger.addHandler(console) frHandler.close() console.close()
class log(object): def __init__(self,fichier,instance): self.logger = logging.getLogger(instance) self.logger.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s :: %(levelname)s :: %(message)s') self.file_handler = RotatingFileHandler(fichier, 'a', 5000000, 1) self.file_handler.setLevel(logging.DEBUG) self.file_handler.setFormatter(formatter) self.file_handler.createLock() self.logger.addHandler(self.file_handler) def ecrire(self,message,niveau): if niveau=='critical': self.logger.critical(message) elif niveau=='error': self.logger.error(message) elif niveau=='warning': self.logger.warning(message) elif niveau=='info': self.logger.info(message) else: self.logger.debug(message) def fermer(self): self.file_handler.close()
class MultiProcessingLog(logging.Handler): def __init__(self, filename, mode="timed", max_bytes=0, backup_count=0): logging.Handler.__init__(self) self.queue = multiprocessing.Queue(-1) if mode == "timed": self._handler = TimedRotatingFileHandler(filename, "midnight", 1, backup_count) else: self._handler = RotatingFileHandler(filename, "a", max_bytes, backup_count) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while self.queue.empty() == False: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
class MultiProcessingLog(logging.Handler): def __init__(self, name, mode = 'a', maxsize = 0, rotate = 0): ''' By default the RotatingFileHandler is set as an usual Handler (no rotating, no maxsize) ''' logging.Handler.__init__(self) self._handler = RotatingFileHandler(name, mode, maxsize, rotate) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
class LoggerSetup: ''' This class sets up the Logger for the project. Ideally, no Print statements should be in the code so that automatic scripts can run in the background and logs can be generated in a file. Also a logging channel for stdout is created for runtime printing of the information/debug messages. ''' def __init__(self, TAG, MAX_FILE_SIZE, BACKUP_COUNT, FILE_LOG_LEVEL, CONSOLE_LOG_LEVEL): ''' Initialization of the logging class. :param TAG: TAG to appear in every logged message line. Any custom string :param MAX_FILE_SIZE: After this file size (in bytes) the log file will be rotated and previous file be backedup :param BACKUP_COUNT: Number of previous logfiles to backup after the size limit is reached on the logs :param FILE_LOG_LEVEL: The log level to be logged in the file (Like logging.NOTSET, logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL) :param CONSOLE_LOG_LEVEL:The log level to be logged in the stdout (Like logging.NOTSET, logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL) ''' self.TAG = TAG self.MAX_FILE_SIZE = MAX_FILE_SIZE self.BACKUP_COUNT = BACKUP_COUNT self.FILE_LOG_LEVEL = FILE_LOG_LEVEL self.CONSOLE_LOG_LEVEL = CONSOLE_LOG_LEVEL def run(self): ''' Just sets up the logger and returns the logger instance which was setup with the init function. :return: logger instance ''' LOG_TAG = 'HUW_XML_PARSER' self.myLogger = logging.getLogger(self.TAG) self.myLogger.setLevel(logging.DEBUG) self.fh = RotatingFileHandler(self.TAG + ".log", 'a', maxBytes=self.MAX_FILE_SIZE, backupCount=self.BACKUP_COUNT) self.fh.setLevel(self.FILE_LOG_LEVEL) self.ch = logging.StreamHandler() self.ch.setLevel(self.CONSOLE_LOG_LEVEL) formatter = logging.Formatter( '[ %(asctime)s ] [ %(name)s ][ %(levelname)s ] %(message)s') self.ch.setFormatter(formatter) self.fh.setFormatter(formatter) self.myLogger.addHandler(self.ch) self.myLogger.addHandler(self.fh) return self.myLogger def close(self): try: self.fh.close() self.ch.close() except Exception as e: self.myLogger.error( "Error while trying to close logger. Maybe this will not be reported :)" )
class MultiprocessingRotatingFileHandler(logging.Handler): def __init__(self, *args, **kwargs): logging.Handler.__init__(self) self._handler = RotatingFileHandler(*args, **kwargs) self.queue = multiprocessing.Queue() t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
def monitor_combine_parameter(): online_combine_log = LOGS_PATH + "/online_combine.log" logger = logging.getLogger() handler = RotatingFileHandler(online_combine_log, maxBytes=1024 * 1024 * 1024, backupCount=3) logger.addHandler(handler) pool = redis.ConnectionPool(host=REDIS_HOST, port=REDIS_PORT, db=0) r = redis.StrictRedis(connection_pool=pool) p = r.pubsub() p.subscribe(TOPIC_HEARTBEAT) while True: for i_item in p.listen(): if 'data' in i_item: if type(i_item['data']) is str: a = eval(i_item['data']) province_id = str(a['province_id']) isp_id = str(a['isp_id']) timestamp = long(a['timestamp']) peer_id = str(a['peer_id']) user_id = peer_id[0:8] nat = int(a['nat_type']) sdk_version = str(a['sdk_version']) combine = { "timestamp": timestamp, "peer_id": peer_id, "user": user_id, "nat_type": nat, "sdk_version": sdk_version, "province_id": province_id, "isp_id": isp_id, "province&isp": "{0}&{1}".format(province_id, isp_id), "province&user": "******".format(province_id, user_id), "province&version": "{0}&{1}".format(province_id, sdk_version), "user&sdk": "{0}&{1}".format(user_id, sdk_version), "sdk&nat": "{0}&{1}".format(sdk_version, nat) } combine_json = json.dumps(combine) logger.warning(combine_json) handler.close()
def register_logging(app): app.logger.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') file_handler = RotatingFileHandler(os.path.join(basedir, "logs/yutou.log"), maxBytes=10 * 1024 * 1024, backupCount=10) file_handler.setFormatter(formatter) file_handler.setLevel(logging.INFO) if not app.debug: app.logger.addHandler(file_handler) else: file_handler.close()
def __console(self, level, message): self.logger = logging.getLogger(self.fileName) self.logger.setLevel(logging.DEBUG) if self.handler is not None: self.logger.addHandler(self.handler) formatter = logging.Formatter( '[%(asctime)s] [%(filename)s:%(funcName)s:%(lineno)d] [%(levelname)s]- %(message)s' ) # 日志输出格式 # 创建一个FileHandler,用于写到本地 fh = RotatingFileHandler( filename=self.logName, mode='a', maxBytes=1024 * 1024 * 5, backupCount=5, encoding='utf-8') # 使用RotatingFileHandler类,滚动备份日志 fh.setLevel(logging.INFO) fh.setFormatter(formatter) self.logger.addHandler(fh) # 创建一个StreamHandler,用于输出到控制台 formatter = colorlog.ColoredFormatter( '%(log_color)s[%(asctime)s] [%(filename)s:%(funcName)s:%(lineno)d] [%(levelname)s]- %(message)s', log_colors=log_colors_config) # 日志输出格式 ch = colorlog.StreamHandler() ch.setLevel(logging.DEBUG) ch.setFormatter(formatter) self.logger.addHandler(ch) if level == 'info': with up_stacked_logger(self.logger, n=2) as logger: logger.info(message) elif level == 'debug': with up_stacked_logger(self.logger, n=2) as logger: logger.debug(message) elif level == 'warning': with up_stacked_logger(self.logger, n=2) as logger: logger.warning(message) elif level == 'error': with up_stacked_logger(self.logger, n=2) as logger: logger.error(message) # 这两行代码是为了避免日志输出重复问题 self.logger.removeHandler(ch) self.logger.removeHandler(fh) if self.handler is not None: self.logger.removeHandler(self.handler) fh.close() # 关闭打开的文件
def logger(logname, logmessage, logfile = 'log.log', loglevel = 'INFO'): try: info = {'CRITICAL': 50, 'ERROR': 40, 'WARNING': 30, 'INFO': 20, 'DEBUG': 10} loglevel = info[loglevel.upper()] handler = RotatingFileHandler(logfile, maxBytes = 1024*1024*100, backupCount = 3) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) logger = logging.getLogger(logname) logger.addHandler(handler) logger.setLevel(logging.DEBUG) logger.log(loglevel, logmessage) finally: logger.removeHandler(handler) handler.close() del logger
def __console(self, level, message): # if self.parent.splitter.splitterState == SplitterState.expanded: # self.handler.new_record.connect(self.textEdit.appendPlainText) if self.handler is not None: self.logger.addHandler(self.handler) formatter = logging.Formatter( '[%(asctime)s] [%(filename)s:%(lineno)d] [%(module)s:%(funcName)s] [%(levelname)s]- %(message)s' ) # 日志输出格式 # 创建一个FileHandler,用于写到本地 fh = RotatingFileHandler( filename=self.logName, mode='a', maxBytes=1024 * 1024 * 5, backupCount=5, encoding='utf-8') # 使用RotatingFileHandler类,滚动备份日志 fh.setLevel(logging.INFO) fh.setFormatter(formatter) self.logger.addHandler(fh) # 创建一个StreamHandler,用于输出到控制台 formatter = colorlog.ColoredFormatter( '%(log_color)s[%(asctime)s] [%(filename)s:%(lineno)d] [%(module)s:%(funcName)s] [%(levelname)s]- %(message)s', log_colors=log_colors_config) # 日志输出格式 ch = colorlog.StreamHandler() ch.setLevel(logging.DEBUG) ch.setFormatter(formatter) self.logger.addHandler(ch) if level == 'info': self.logger.info(message) elif level == 'debug': self.logger.debug(message) elif level == 'warning': self.logger.warning(message) elif level == 'error': self.logger.error(message) # 这两行代码是为了避免日志输出重复问题 self.logger.removeHandler(ch) self.logger.removeHandler(fh) if self.handler is not None: self.logger.removeHandler(self.handler) # if self.parent.splitter.splitterState == SplitterState.expanded: # self.handler.new_record.disconnect(self.textEdit.appendPlainText) fh.close() # 关闭打开的文件
class SmoresFileLog(logging.Handler): def __init__(self, name, mode, maxsize=0, backup=1): global handler_list logging.Handler.__init__(self) self._handler = RotatingFileHandler(name, mode, maxBytes=maxsize, backupCount=backup, encoding='UTF-8') if self.name not in handler_list: handler_list.append(name) try: self._handler.doRollover() except (PermissionError, ValueError): pass def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def _format_record(self, record): # ensure that exc_info and args have been stringified. Removes any # chance of unpickleable things inside and possibly reduces message size # sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self._handler.emit(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
class Log(object): def __init__(self, log_id, log_path, log_level=logging.INFO): self.logger_ = logging.getLogger(log_id) self._handler = RotatingFileHandler(log_path, mode='a', maxBytes=100 * 1024 * 1024, backupCount=2) self._handler.setFormatter(logging.Formatter("%(asctime)s-%(filename)s:%(lineno)d[%(levelname)s]:%(message)s")) self.logger_.setLevel(log_level) self._handler.setLevel(log_level) self.logger_.addHandler(self._handler) def __del__(self): self._handler.close() def setdebuglevel(self): self.logger_.setLevel(logging.DEBUG) def setinfolevel(self): self.logger_.setLevel(logging.INFO) @property def log(self): return self.logger_
def monitor_lsm_free(): lsm_free_log = LOGS_PATH + "/sdk_lsm_free.log" logger = logging.getLogger('lsm_free') handler = RotatingFileHandler(lsm_free_log, maxBytes=1024 * 1024 * 1024, backupCount=3) logger.addHandler(handler) pool = redis.ConnectionPool(host=REDIS_HOST, port=REDIS_PORT, db=0) r = redis.StrictRedis(connection_pool=pool) p = r.pubsub() p.subscribe(TOPIC_VOD_LSM) while True: for i_item in p.listen(): if 'data' in i_item: if type(i_item['data']) is str: a = eval(i_item['data']) # lsm_free unit : B ---> MB lsm_free = int(a['lsm_free']) / 1024 / 1024 if lsm_free <= 10: lsm_free = LSM_FREE_LEVEL_A elif 10 < lsm_free <= 50: lsm_free = LSM_FREE_LEVEL_B elif 50 < lsm_free <= 100: lsm_free = LSM_FREE_LEVEL_C elif 100 < lsm_free <= 300: lsm_free = LSM_FREE_LEVEL_D elif 300 < lsm_free <= 600: lsm_free = LSM_FREE_LEVEL_E else: lsm_free = LSM_FREE_LEVEL_F timestamp = long(a['timestamp']) peer_id = str(a['peer_id']) combine = { "timestamp": timestamp, "peer_id": peer_id, "lsm_level": lsm_free } combine_json = json.dumps(combine) logger.warning(combine_json) handler.close()
def monitor_p2p_ratio(): online_combine_log = LOGS_PATH + "/p2p_ratio.log" logger = logging.getLogger() handler = RotatingFileHandler(online_combine_log, maxBytes=1024 * 1024 * 1024, backupCount=3) logger.addHandler(handler) pool = redis.ConnectionPool(host=REDIS_HOST, port=REDIS_PORT, db=0) r = redis.StrictRedis(connection_pool=pool) p = r.pubsub() p.subscribe(TOPIC_FLOW) while True: for i_item in p.listen(): if 'data' in i_item: if type(i_item['data']) is str: p2p_data = eval(i_item['data']) timestamp = long(p2p_data['timestamp']) p2p_download = int(p2p_data['p2p_download']) cdn_download = int(p2p_data['cdn_download']) file_id = str(p2p_data['file_id']) if p2p_download > 0 or cdn_download > 0: p2p_combine = { "timestamp": timestamp, "p2p_download": p2p_download, "cdn_download": cdn_download, "file_id": file_id, "p2p_ratio": float(p2p_download * 100 / (p2p_download + cdn_download)) } combine_json = json.dumps(p2p_combine) logger.warning(combine_json) handler.close()
class FileCollector(object): def __init__(self, remote_ip, remote_dir='/letv/crawler_delta', local_dir='./in'): self._remote_ip = remote_ip self._remote_dir = remote_dir self._local_dir = local_dir self._init_dir() self._init_log() def _init_dir(self): if not os.path.exists(self._local_dir): os.mkdir(self._local_dir) def _init_log(self): log_name = "file_collector_%s.error" % self._remote_ip self._handler = RotatingFileHandler(log_name, mode='a', maxBytes=100 * 1024 * 1024, backupCount=2) self._handler.setFormatter(logging.Formatter('[%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s')) #self._handler.setLevel(logging.DEBUG) self._logger = logging.getLogger(log_name) self._logger.addHandler(self._handler) self._logger.setLevel(logging.DEBUG) def __del__(self): self._handler.close() self._logger.removeHandler(self._handler) def _ls_file_list(self): cmd = 'ssh %s ls %s' % (self._remote_ip, self._remote_dir) #self._logger.debug('excute cmd:%s', cmd) sta, result = call_cmd(cmd) #print sta, result if sta == 0: #self._logger.debug('success get filelist:\n%s', result) return result.split() else: return None def _scp_file(self, filename, filename_tmp): cmd = 'scp search@%s:%s/%s %s/%s' % (self._remote_ip, self._remote_dir, filename, self._local_dir, filename_tmp) #self._logger.debug('excute cmd:%s', cmd) sta, result = call_cmd(cmd) return True if sta==0 else False def _delete_file_remote(self, filename): cmd = 'ssh search@%s rm %s/%s' % (self._remote_ip, self._remote_dir, filename) #self._logger.debug('excute cmd:%s', cmd) sta, result = call_cmd(cmd) return True if sta == 0 else False def _rename_file(self, filename, filename_tmp): if not filename or not filename_tmp: return local_filename = '%s/%s' % (self._local_dir, filename) local_filename_tmp = '%s/%s' % (self._local_dir, filename_tmp) try: os.rename(local_filename_tmp, local_filename) except Exception: self._logger.exception('failed to rename %s to %s' % (local_filename_tmp, local_filename)) def run(self): file_list = self._ls_file_list() if not file_list: return for filename in file_list: if not filename.endswith('.txt'): continue filename_tmp = '%s_copying' % filename if self._scp_file(filename, filename_tmp): self._rename_file(filename, filename_tmp) self._logger.debug('success scp file filename:[%s]', filename) if self._delete_file_remote(filename): self._logger.debug('success delete remote file filename:[%s]', filename) else: self._logger.error('failed delete remote file filename:[%s]', filename)
class TaskProcess(Process): """Process taking care of performing the measures. When started this process sets up a logger redirecting all records to a queue. It then redirects stdout and stderr to the logging system. Then as long as it is not stopped it waits for the main process to send a measures through the pipe. Upon reception of the `ConfigObj` object describing the measure it rebuilds it, set up a logger for that specific measure and if necessary starts a spy transmitting the value of all monitored entries to the main process. It finally run the checks of the measure and run it. It can be interrupted by setting an event and upon exit close the communication pipe and signal all listeners that it is closing. Parameters ---------- pipe : double ended multiprocessing pipe Pipe used to communicate with the parent process which is transferring the measure to perform. log_queue : multiprocessing queue Queue in which all log records are sent to be procesed later in the main process. monitor_queue : multiprocessing queue Queue in which all the informations the user asked to monitor during the measurement are sent to be processed in the main process. task_pause : multiprocessing event Event set when the user asked the running measurement to pause. task_paused : multiprocessing event Event set when the current measure is paused. task_stop : multiprocessing event Event set when the user asked the running measurement to stop. process_stop : multiprocessing event Event set when the user asked the process to stop. Attributes ---------- meas_log_handler : log handler Log handler used to save the running measurement specific records. see `Parameters` Methods ------- run(): Method called when the new process starts. """ def __init__(self, pipe, log_queue, monitor_queue, task_pause, task_paused, task_stop, process_stop): super(TaskProcess, self).__init__(name='MeasureProcess') self.daemon = True self.task_pause = task_pause self.task_paused = task_paused self.task_stop = task_stop self.process_stop = process_stop self.pipe = pipe self.log_queue = log_queue self.monitor_queue = monitor_queue self.meas_log_handler = None def run(self): """Method called when the new process starts. For a complete description of the workflow see the class docstring. """ self._config_log() # Ugly patch to avoid pyvisa complaining about missing filters warnings.simplefilter("ignore") # Redirecting stdout and stderr to the logging system. logger = logging.getLogger() redir_stdout = StreamToLogRedirector(logger) sys.stdout = redir_stdout redir_stderr = StreamToLogRedirector(logger, 'stderr') sys.stderr = redir_stderr logger.info('Logger parametrised') logger.info('Process running') self.pipe.send('READY') while not self.process_stop.is_set(): # Prevent us from crash if the pipe is closed at the wrong moment. try: # Wait for a measurement. while not self.pipe.poll(2): if self.process_stop.is_set(): break if self.process_stop.is_set(): break # Get the measure. name, config, build, runtime, mon_entries = self.pipe.recv() # Build it by using the given build dependencies. root = build_task_from_config(config, build, True) # Give all runtime dependencies to the root task. root.run_time = runtime logger.info('Task built') # There are entries in the database we are supposed to # monitor start a spy to do it. if mon_entries: spy = MeasureSpy( self.monitor_queue, mon_entries, root.task_database) # Set up the logger for this specific measurement. if self.meas_log_handler is not None: logger.removeHandler(self.meas_log_handler) self.meas_log_handler.close() self.meas_log_handler = None log_path = os.path.join( root.get_from_database('default_path'), name + '.log') if os.path.isfile(log_path): os.remove(log_path) self.meas_log_handler = RotatingFileHandler(log_path, mode='w', maxBytes=10**6, backupCount=10) aux = '%(asctime)s | %(levelname)s | %(message)s' formatter = logging.Formatter(aux) self.meas_log_handler.setFormatter(formatter) logger.addHandler(self.meas_log_handler) # Pass the events signaling the task it should stop or pause # to the task and make the database ready. root.should_pause = self.task_pause root.paused = self.task_paused root.should_stop = self.task_stop root.task_database.prepare_for_running() # Perform the checks. check, errors = root.check(test_instr=True) # They pass perform the measure. if check: logger.info('Check successful') root.perform_(root) result = ['', '', ''] if self.task_stop.is_set(): result[0] = 'INTERRUPTED' result[2] = 'Measure {} was stopped'.format(name) else: result[0] = 'COMPLETED' result[2] = 'Measure {} succeeded'.format(name) if self.process_stop.is_set(): result[1] = 'STOPPING' else: result[1] = 'READY' self.pipe.send(tuple(result)) # They fail, mark the measure as failed and go on. else: mes = 'Tests failed, see log for full records.' self.pipe.send(('FAILED', 'READY', mes)) # Log the tests that failed. fails = errors.iteritems() message = '\n'.join('{} : {}'.format(path, mes) for path, mes in fails) logger.critical(message) # If a spy was started kill it if mon_entries: spy.close() del spy except IOError: pass # Clean up before closing. logger.info('Process shuting down') if self.meas_log_handler: self.meas_log_handler.close() self.log_queue.put_nowait(None) self.monitor_queue.put_nowait((None, None)) self.pipe.close() def _config_log(self): """Configuring the logger for the process. Sending all record to a multiprocessing queue. """ config_worker = { 'version': 1, 'disable_existing_loggers': True, 'handlers': { 'queue': { 'class': 'hqc_meas.utils.log.tools.QueueHandler', 'queue': self.log_queue, }, }, 'root': { 'level': 'DEBUG', 'handlers': ['queue'] }, } logging.config.dictConfig(config_worker)
class TaskProcess(Process): """Process taking care of performing the measures. When started this process sets up a logger redirecting all records to a queue. It then redirects stdout and stderr to the logging system. Then as long as there is measures to perform it asks the main process to send it measures through a pipe. Upon reception of the `ConfigObj` object describing the measure it rebuilds it, set up a logger for that specific measure and if necessary starts a spy transmitting the value of all monitored entries to the main process. It finally run the checks of the measure and run it. It can be interrupted by setting an event and upon exit close the communication pipe and signal all listeners that it is closing. Parameters ---------- pipe : double ended multiprocessing pipe Pipe used to communicate with the parent process which is transferring the measure to perform. log_queue : multiprocessing queue Queue in which all log records are sent to be procesed later in the main process. monitor_queue : multiprocessing queue Queue in which all the informations the user asked to monitor during the measurement are sent to be processed in the main process. task_stop : multiprocessing event Event set when the user asked the running measurement to stop. process_stop : multiprocessing event Event set when the user asked the process to stop. Attributes ---------- meas_log_handler : log handler Log handler used to save the running measurement specific records. see `Parameters` Methods ------- run(): Method called when the new process starts. """ def __init__(self, pipe, log_queue, monitor_queue, task_stop, process_stop): super(TaskProcess, self).__init__(name = 'MeasureProcess') self.task_stop = task_stop self.process_stop = process_stop self.pipe = pipe self.log_queue = log_queue self.monitor_queue = monitor_queue self.meas_log_handler = None def run(self): """Method called when the new process starts. For a complete description of the workflow see the class docstring. """ self._config_log() # Ugly patch to avoid pyvisa complaining about missing filters warnings.simplefilter("ignore") # Redirecting stdout and stderr to the logging system. logger = logging.getLogger() redir_stdout = StreamToLogRedirector(logger) sys.stdout = redir_stdout logger.info('Logger parametrised') print 'Process running' while not self.process_stop.is_set(): try: # Request a new measure to perform from the main process print 'Need task' self.pipe.send('Need task') # Get the answer self.pipe.poll(None) name, config, monitored_entries = self.pipe.recv() if config != 'STOP': # If a real measurement was sent, build it. task = IniConfigTask().build_task_from_config(config) print 'Task built' # There are entries in the database we are supposed to # monitor start a spy to do it. if monitored_entries is not None: spy = MeasureSpy(self.monitor_queue, monitored_entries, task.task_database) # Set up the logger for this specific measurement. if self.meas_log_handler != None: logger.removeHandler(self.meas_log_handler) self.meas_log_handler.close() self.meas_log_handler = None log_path = os.path.join( task.get_from_database('default_path'), name + '.log') if os.path.isfile(log_path): os.remove(log_path) self.meas_log_handler = RotatingFileHandler(log_path, mode = 'w', maxBytes = 10**6, backupCount = 10) aux = '%(asctime)s | %(levelname)s | %(message)s' formatter = logging.Formatter(aux) self.meas_log_handler.setFormatter(formatter) logger.addHandler(self.meas_log_handler) # Clear the event signaling the task it should stop, pass it # to the task and make the database ready. self.task_stop.clear() task.should_stop = self.task_stop task.task_database.prepare_for_running() # Perform the checks. check = task.check(test_instr = True) if check[0]: print 'Check successful' # Perform the measure task.process() self.pipe.send('Task processed') if self.task_stop.is_set(): print 'Task interrupted' else: print 'Task processed' else: message = '\n'.join('{} : {}'.format(path, mes) for path, mes in check[1].iteritems()) logger.critical(message) # If a spy was started kill it if monitored_entries is not None: spy.close() del spy except IOError: pass # Clean up before closing. self.pipe.send('Closing') print 'Process shuting down' if self.meas_log_handler: self.meas_log_handler.close() self.log_queue.put_nowait(None) self.pipe.close() def _config_log(self): """Configuring the logger for the process. Sending all record to a multiprocessing queue. """ config_worker = { 'version': 1, 'disable_existing_loggers': True, 'handlers': { 'queue': { 'class': 'hqc_meas.log_facility.QueueHandler', 'queue': self.log_queue, }, }, 'root': { 'level': 'DEBUG', 'handlers': ['queue'] }, } logging.config.dictConfig(config_worker) if os.name == 'posix': # On POSIX, the setup logger will have been configured in the # parent process, but should have been disabled following the # dictConfig call. # On Windows, since fork isn't used, the setup logger won't # exist in the child, so it would be created and the message # would appear - hence the "if posix" clause. logger = logging.getLogger('setup') logger.critical('Should not appear, because of disabled logger ...')
'-distro.html':self.get_distro_specific_html(), '.rst':'\n'.join(map(to_ascii, self.getProfile())), } logdir = os.path.expanduser('~/.smolt/') try: if not os.path.exists(logdir): os.mkdir(logdir, 1700) for k, v in log_matrix.items(): filename = os.path.expanduser(os.path.join( logdir, 'submission%s' % k)) r = RotatingFileHandler(filename, \ maxBytes=1000000, backupCount=9) r.stream.write(v) r.doRollover() r.close() os.remove(filename) except: pass del logdir del log_matrix debug('sendHostStr: %s' % serialized_host_obj_machine) debug('Sending Host') if batch: entry_point = "/client/batch_add_json" logging.debug('Submitting in asynchronous mode') else: entry_point = "/client/add_json"
class MultiprocessingLogHandler(logging.Handler): """ by zzzeek on SO: http://stackoverflow.com/questions/641420/how-should-i-log-while-using-multiprocessing-in-python """ def __init__(self, *args, **kwargs): logging.Handler.__init__(self) self.lock = None if 'klass' in kwargs: klass = kwargs['klass'] del kwargs['klass'] self._handler = klass(*args, **kwargs) else: self._handler = RotatingFileHandler(*args, **kwargs) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
class Logger(Singleton): def __init__(self): self.__log = None self.__hdlr = None def __del__(self): self.close() ''' introduction: get instance of loggger @parameter: log_instance: id of logger file_name: name you are going to produce max_bytes: capacity of file backup_count: rollback number of file timeopen: 0 -----close, others -----open return: 0 ----- success -1 ----- failure ''' def get_log(self, log_instance, file_name, max_bytes, backup_count, timeopen = 0): result = 0 try: self.__log = logging.getLogger(log_instance) except: self.__log = None result = -1 else: # print id(self.__log) self.__log.setLevel(logging.DEBUG) try: self.__hdlr = RotatingFileHandler(file_name, maxBytes = max_bytes, backupCount = backup_count) except: self.__hdlr = None result = -1 else: self.__hdlr.setLevel(logging.DEBUG) formatter = None if 0 != timeopen: formatter = logging.Formatter('%(asctime)s - %(message)s') else: formatter = logging.Formatter('%(message)s') self.__hdlr.setFormatter(formatter) self.__log.addHandler(self.__hdlr) finally: return result def write(self, value): if None != self.__log and None != self.__hdlr: self.__log.debug(value) def flush(self): if None != self.__hdlr: self.__hdlr.flush() def close(self): if None != self.__log: self.__log.removeHandler(self.__hdlr) self.__log = None if None != self.__hdlr: self.__hdlr.flush() self.__hdlr.close() self.__hdlr = None
def close(self): _RotatingFileHandler.close(self) _remove_from_reopenable(self._wr)
class MultiProcessingLog(logging.Handler): """ Synchronised parallel multi processing log. """ def __init__(self, name, mode, maxsize, rotate, encoding="utf-8", delay=0): logging.Handler.__init__(self) self._handler = RotatingFileHandler(name, mode, maxsize, rotate, encoding, delay) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): """ Default setFormatter impl. """ logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): """ Receives one record to log. """ while True: try: record = self.queue.get() if self._handler: self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): """ Puts to nowait queue. """ self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: # noinspection PyUnusedLocal dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() self._handler = None logging.Handler.close(self)
class MultiProcessingLogger(logging.Handler): """A multiprocessing-safe logger with built in log rolling/backups""" def __init__(self, name, mode='a', maxsize=10000, rotate=100): """Create a new MultiprocessingLogger for a file named *name* :param name: The name of the log file to be written to :param mode: The mode in which the file should be opened in :param maxsize: The number of bytes this log file should roll over at :param rotate: The number of rolled log files to keep hanging around """ logging.Handler.__init__(self) self._handler = RotatingFileHandler(name, mode, maxsize, rotate) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
try: mgr = LciManager(reactor, config, log) # we ignore SIGTERM because Squid will close the log FH, which gives # us a much cleaner signal that we're to shut down. signal.signal(signal.SIGTERM, signal.SIG_IGN) mgr.start() except ConfigParser.Error, why: error("Configuration file: %s" % why) except Exception, why: error("Error: %s " % why) except: error("Unknown error.") # clean up logging hdlr.flush() hdlr.close() logging.shutdown() def error(msg): "Something really bad has happened. Should only be used during startup." logging.critical(msg) sys.stderr.write("LCI FATAL: %s\n" % msg) sys.exit(1) ############################################################################ class ManagerState: "Holds the manager's state in an easily persistable way." def __init__(self): self.groups = {} # key is hashed group_uri; value is set of req uris