def process_logs(config): """ 扫描日志目录下所有日志文件,将非今日产生的文件非压缩文件进行压缩 计算日志目录中所有日志文件大小,所超过设置的最大日志量,依次删除老日志文件 直到总日志量小于等于设置的最大日志量 :param config: 配置参数 :return: """ for filename in os.listdir(LOG_DIR): filepath = os.path.join(LOG_DIR, filename) if os.path.isfile(filepath) and not filename.endswith( '.zip') and not filename.endswith( '.lock') and filepath != make_log_filename(config): zipfilepath = filepath + '.zip' azip = zipfile.ZipFile(zipfilepath, 'w') azip.write(filepath, compress_type=ZIP_COMPRESSION_MODE) azip.close() os.remove(filepath) total_size = get_dir_total_size(LOG_DIR) max_size = unitstr_to_bytes(config.log.maxsize) if total_size > max_size: files = sorted( os.listdir(LOG_DIR), key=lambda x: os.path.getmtime(os.path.join(LOG_DIR, x))) for file in files: filepath = os.path.join(LOG_DIR, file) size = os.path.getsize(filepath) total_size -= size os.remove(filepath) if total_size <= max_size: break
def create_size_file(self, filename, size): size = unitstr_to_bytes(size) full_path = os.path.join(self.rootdir, filename) with open(full_path, 'w') as fp: fp.seek(size - 1) fp.write('a') fp.close() return full_path
def init_logger(config): """ 初始化日志句柄(写文件) 需要多进程安全的ConcurrentRotatingFileHandler 当日志级别大于等于Warning时会同时打印到屏幕和写文件 :param config: 配置参数 :return: """ # console_handler = logging.StreamHandler(sys.__stdout__) console_handler = logging.StreamHandler(sys.__stderr__) console_handler.level = logging.ERROR # console_handler.level = logging.DEBUG console_logger = logging.getLogger('obscmd') console_logger.addHandler(console_handler) format = '%(asctime)s - %(levelname)s - %(process)d - %(thread)d - %(filename)s[line:%(lineno)d] - %(message)s' logfile = make_log_filename(config) handler = ConcurrentRotatingFileHandler( logfile, mode='a', maxBytes=unitstr_to_bytes(config.log.maxbytes), backupCount=int(config.log.backupcount), encoding=None, delay=0) handler.setFormatter(logging.Formatter(format)) logger = logging.getLogger("obscmd.file") logger.propagate = False logger.addHandler(handler) # logger.setLevel(logging.DEBUG) format = '%(message)s' handler1 = ConcurrentRotatingFileHandler( logfile, mode='a', maxBytes=unitstr_to_bytes(config.log.maxbytes), backupCount=int(config.log.backupcount), encoding=None, delay=0) handler1.setFormatter(logging.Formatter(format)) logger1 = logging.getLogger("print") logger1.propagate = True logger1.addHandler(handler1) logger.setLevel(LOG_LEVEL[config.log.level]) return logger, logfile
def print_run_time(self, size, seconds, tasknum, cmd): total = unitstr_to_bytes(size) flowspeed = bytes_to_unitstr(total / seconds) msg = '%s/s\t\d tasks\t%s' % (flowspeed, tasknum, cmd) print(msg)
def test_bytes_more_gb(self): self.assertEqual(unitstr_to_bytes('3.6G'), int(3.6 * GB))
def test_bytes_mb_lowercase(self): self.assertEqual(unitstr_to_bytes('100M'), int(100 * MB))
def test_bytes_mb_lowercase(self): self.assertEqual(unitstr_to_bytes('23.7m'), int(23.7 * MB))
def test_bytes_kb_lowercase(self): self.assertEqual(unitstr_to_bytes('1.1k'), int(1.1 * KB))
def test_bytes_kb(self): self.assertEqual(unitstr_to_bytes('1K'), int(1024 * 1.0))
def test_bytes_less_kb(self): self.assertEqual(unitstr_to_bytes('1023B'), 1023)
def test_1b(self): self.assertEqual(unitstr_to_bytes('1B'), 1)
def test_no_unit(self): self.assertEqual(unitstr_to_bytes('123133.3'), 123133)