def configure_logger(self):
logger = logging.getLogger()
# Create a handler for STDOUT, but only on the root rank:
if not MPI_AVAILABLE or hvd.rank() == 0:
stream_handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(levelname)s - %(message)s')
stream_handler.setFormatter(formatter)
handler = handlers.MemoryHandler(capacity=1, target=stream_handler)
logger.addHandler(handler)
# Add a file handler:
# Add a file handler too:
log_file = self.config.save_path + "/process.log"
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(formatter)
file_handler = handlers.MemoryHandler(capacity=1,
target=file_handler)
logger.addHandler(file_handler)
logger.setLevel(logging.DEBUG)
# fh = logging.FileHandler('run.log')
# fh.setLevel(logging.DEBUG)
# logger.addHandler(fh)
else:
# in this case, MPI is available but it's not rank 0
# create a null handler
handler = logging.NullHandler()
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
def configure_logger(rank):
'''Configure a global logger
Adds a stream handler and a file hander, buffers to file (10 lines) but not to stdout.
Submit the MPI Rank
'''
logger = logging.getLogger()
# Create a handler for STDOUT, but only on the root rank.
# If not distributed, we still get 0 passed in here.
if rank == 0:
stream_handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(levelname)s - %(message)s')
stream_handler.setFormatter(formatter)
handler = handlers.MemoryHandler(capacity=0, target=stream_handler)
logger.addHandler(handler)
# Add a file handler too:
log_file = "process.log"
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(formatter)
file_handler = handlers.MemoryHandler(capacity=10, target=file_handler)
logger.addHandler(file_handler)
logger.setLevel(logging.INFO)
else:
# in this case, MPI is available but it's not rank 0
# create a null handler
handler = logging.NullHandler()
logger.addHandler(handler)
logger.setLevel(logging.INFO)
def configure_logger(self, rank):
logger = logging.getLogger()
# Create a handler for STDOUT, but only on the root rank.
# If not distributed, we still get 0 passed in here.
if rank == 0:
stream_handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(levelname)s - %(message)s')
stream_handler.setFormatter(formatter)
handler = handlers.MemoryHandler(capacity=0, target=stream_handler)
logger.addHandler(handler)
# Add a file handler too:
log_file = self.args.output_dir + "/process.log"
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(formatter)
file_handler = handlers.MemoryHandler(capacity=10,
target=file_handler)
logger.addHandler(file_handler)
logger.setLevel(logging.INFO)
else:
# in this case, MPI is available but it's not rank 0
# create a null handler
handler = logging.NullHandler()
logger.addHandler(handler)
logger.setLevel(logging.INFO)
def configure_logger():
'''Configure a global logger
Adds a stream handler and a file hander, buffers to file (10 lines) but not to stdout.
Submit the MPI Rank
'''
logger = logging.getLogger()
# Create a handler for STDOUT, but only on the root rank.
stream_handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
stream_handler.setFormatter(formatter)
handler = handlers.MemoryHandler(capacity=0, target=stream_handler)
logger.addHandler(handler)
# Add a file handler too:
log_file = "inference.log"
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(formatter)
file_handler = handlers.MemoryHandler(capacity=10, target=file_handler)
logger.addHandler(file_handler)
logger.setLevel(logging.INFO)
def initialize(stats_server="localhost", stats_port=514):
"""
Initialize ACS logger
"""
# stdout handler configuration
ACSLogging.stdout_handler = logging.StreamHandler(sys.stdout)
ACSLogging.stdout_handler.setFormatter(ACSLogging.formater)
ACSLogging.stdout_handler.setLevel(logging.DEBUG)
# memory handler to store logs waiting for file handler creation
ACSLogging.memory_handler = handlers.MemoryHandler(1024 * 2)
ACSLogging.memory_handler.setFormatter(ACSLogging.formater)
ACSLogging.memory_handler.setLevel(logging.DEBUG)
acs_logger = logging.getLogger(ACS_LOGGER_NAME)
acs_logger.setLevel(logging.DEBUG)
acs_logger.handlers = []
acs_logger.addHandler(ACSLogging.stdout_handler)
acs_logger.addHandler(ACSLogging.memory_handler)
acs_logger.name = ACS_LOGGER_NAME
logging.addLevelName(RAW_LEVEL, "RAW")
logging.addLevelName(ACSLogging.MINIMAL_LEVEL, "MINIMAL")
LOGGER_STATS.setLevel(logging.DEBUG)
LOGGER_STATS.propagate = False
ACSLogging.sys_log_handler = EnhSysLogHandler(address=(stats_server, stats_port))
ACSLogging.sys_log_handler.setFormatter(AltCustomFormatter(STATS_FORMATER_CONF, FORMATER_DATE_CONF))
LOGGER_STATS.addHandler(ACSLogging.sys_log_handler)
def init_log(self, name, level, host, port, capacity=100):
target = handlers.SocketHandler(host, port)
if capacity > 0:
hdlr = handlers.MemoryHandler(capacity, logging.ERROR, target)
else:
hdlr = target
hdlr.setLevel(level)
self.logger = logging.getLogger(name)
self.logger.addHandler(hdlr)
self.logger.setLevel(level)
def get_logger(name, level, host, port, capacity=1024):
target = handlers.SocketHandler(host, port)
if capacity > 0:
hdlr = handlers.MemoryHandler(capacity, logging.ERROR, target)
else:
hdlr = target
hdlr.setLevel(level)
logger = logging.getLogger(name)
logger.addHandler(hdlr)
logger.setLevel(level)
return logger
def test_divide_by_zero(self):
memory_handler = handlers.MemoryHandler(256)
root_logger = logging.getLogger()
root_logger.handlers = []
root_logger.addHandler(memory_handler)
script = """
assign a {1 / 0}
"""
self._runner.run_script(script)
self.assertTrue('division by zero' in memory_handler.buffer[0].msg)
def setUp(self):
"""Sets up logging output for assert checks."""
log_entries = self._log_entries = []
class Target(object):
"""The target handler to store log output."""
def handle(self, record):
log_entries.append(
(record.levelname, record.msg % record.args))
self._handler = handlers.MemoryHandler(capacity=0, target=Target())
logging.getLogger().addHandler(self._handler)
def list(self, request, *args, **kwargs):
target = log.handlers
log.info("测试log")
print("当前handlers")
print(target)
memory_handler = handlers.MemoryHandler(
capacity=1, flushLevel=logging.WARNING, target=target)
innerlog = logging.getLogger("nothing")
innerlog.addHandler(memory_handler)
innerlog.warning("warning日志")
innerlog.info("info日志")
innerlog.info(__name__)
print(innerlog.handlers)
return super(TestLogViewSet, self).list(request, *args, **kwargs)
def getSocketLogger(name, level, host, port, memoryCapacity):
target = handlers.SocketHandler(host, port)
if memoryCapacity > 0:
hdlr = handlers.MemoryHandler(
memoryCapacity,
logging.ERROR, # 此参数是指遇到此级别时，马上flush
target)
else:
hdlr = target
hdlr.setLevel(level)
logger = logging.getLogger(name)
logger.addHandler(hdlr)
logger.setLevel(level)
return logger
def buffered_hdlr(target=None, capacity=4096, level='error', **kwargs):
"""A memory buffered log handler
Args:
target (obj): The target logger handler (default stdout).
capacity (int): The buffer size (default 4096).
level (string): The min event level required to flush buffer
(default: error).
Returns:
New instance of :class:`logging.handlers.MemoryHandler`
Examples:
>>> buffered_hdlr() # doctest: +ELLIPSIS
<logging.handlers.MemoryHandler object at 0x...>
"""
target = target or logging.StreamHandler(sys.stdout)
return hdlrs.MemoryHandler(capacity, level.upper(), target)
def addHandler(self, handler):
if handler not in self.handlers:
self.handlers.append(handler)
def SetLevel(self, level):
if type(level) == type(CRITICAL):
self.level = level
else:
logging.getLogger(Logging).error("Invalid level type: " +
str(type(level)))
return
# Make sure handler is accepting at this level or with more detail.
for handler in self.handlers:
if handler.level > level: # closer to zero means more detail
handler.setLevel(level)
# 0. Initialize logging, storing in log data memory
defStreamHandler = StreamHandler()
defStreamHandler.setFormatter(GetFormatter())
fn = tempfile.mktemp(".aglog")
defLogHandler = FileHandler(fn)
defLogHandler.setFormatter(GetFormatter())
# This is set to 4096 to get around pythonw limitations!!!
# http://mail.python.org/pipermail/python-list/2004-June/227268.html
mlh = handlers.MemoryHandler(16384, flushLevel=CRITICAL, target=defLogHandler)
mlh.setFormatter(GetFormatter())
memLevels = HandleLoggers(mlh, GetDefaultLoggers())
memLevels.SetLevel(DEBUG)
def create_logger(slack=True, file=False, config: dict=None):
if slack:
slack_handler = handlers.MemoryHandler(1, flushLevel=logging.INFO, target=SlackLogStreamer())
LOGGER.addHandler(slack_handler)
return LOGGER
def memoryWapper(handler, capacity):
hdlr = lhandlers.MemoryHandler(capacity, target=handler)
hdlr.setFormatter(handler.formatter)
return hdlr
for i in logger.handlers:
logger.removeHandler(i)
file_handler = logging.FileHandler(LogName, encoding='utf-8')
file_handler.setFormatter(formatter)
file_handler.setLevel(logging.DEBUG)
# 控制台日志Handler
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(formatter)
console_handler.setLevel(logging.INFO)
# 邮件日志Handler
mail_handler = handlers.SMTPHandler(
("smtp.163.com", 25), '*****@*****.**',
['*****@*****.**', '*****@*****.**'],
"logging from my app",
credentials=('*****@*****.**', ""),
)
mail_handler.setFormatter(formatter)
mail_handler.setLevel(logging.ERROR)
memory_handler = handlers.MemoryHandler(100)
memory_handler.setFormatter(formatter)
memory_handler.setLevel(logging.ERROR)
memory_handler.setTarget(mail_handler)
# 添加Handler生效
logger.addHandler(console_handler)
logger.addHandler(file_handler)
# logger.addHandler(memory_handler)
def _memHandler(self):
'''Defines a memory handler'''
self.memhandler = handlers.MemoryHandler(capacity=1024)
self.memhandler.setFormatter(self.format)
self.log.addHandler(self.memhandler)
