def setup_logging(filename,
level=levels.DEBUG,
format='json',
redirect=True,
format_string=_default_line_formatter):
if filename is None:
if format == 'json':
args = {}
if filename:
args['filename'] = filename
else:
args['stream'] = sys.stdout
output = JsonOutput(**args)
else:
output = outputs.StreamOutput(format=format_string)
else:
if format == 'json':
output = JsonOutput(filename)
else:
output = outputs.FileOutput(filename, format=format_string)
add_emitters(('emitter', level, None, output))
if redirect:
# redirect standart logging to twiggy
del logging.root.handlers[:]
logging.root.level = 0 # to pass all messages to redirect handler
logging.root.addHandler(RedirectLoggingHandler())
def test_sanity(self):
from twiggy import add_emitters, log
from twiggy.logging_compat import LoggingBridgeOutput, DEBUG
logger = log.name("decoy")
add_emitters(("decoy", DEBUG, None, LoggingBridgeOutput()))
logger.error("spam")
logger.notice("eggs")
def test_format(self):
logger = log.name("spam")
list_output = ListOutput(format=LoggingBridgeFormat())
messages = list_output.messages
add_emitters(("spam", DEBUG, None, list_output))
logger.error("eggs")
self.failUnlessEqual(messages[0], ('|eggs\n', ERROR, 'spam'))
def setup_logging(filename,
level=levels.DEBUG,
format='json',
redirect=True,
format_string=_default_line_formatter):
if filename is None:
if format == 'json':
args = {}
if filename:
args['filename'] = filename
else:
args['stream'] = sys.stdout
output = JsonOutput(**args)
else:
output = outputs.StreamOutput(format=format_string)
else:
if format == 'json':
output = JsonOutput(filename)
else:
output = outputs.FileOutput(filename, format=format_string)
add_emitters(('emitter', level, None, output))
if redirect:
# redirect standart logging to twiggy
del logging.root.handlers[:]
logging.root.level = 0 # to pass all messages to redirect handler
logging.root.addHandler(RedirectLoggingHandler())
def setUp(self):
from twiggy import add_emitters
from twiggy.logging_compat import getLogger, DEBUG
from twiggy.outputs import ListOutput
self.logger = getLogger("spam")
self.logger.setLevel(DEBUG)
self.list_output = ListOutput()
self.messages = self.list_output.messages
add_emitters(("spam", DEBUG, None, self.list_output))
def test_integration(self):
all_ = twiggy.outputs.StreamOutput(stream=StringIO(),
format=twiggy.formats.line_format)
out1 = twiggy.outputs.StreamOutput(stream=StringIO(),
format=twiggy.formats.line_format)
out2 = twiggy.outputs.StreamOutput(stream=StringIO(),
format=twiggy.formats.line_format)
twiggy.add_emitters(
('*', twiggy.levels.DEBUG, None, all_),
('first', twiggy.levels.INFO, None, out1),
('second', twiggy.levels.DEBUG,
twiggy.filters.glob_names('second.*'), out2),
('first-filter', twiggy.levels.DEBUG, ".*pants.*", out1))
def something():
return "something cool"
twiggy.log.debug("oh hi")
twiggy.log.name("second").info("do you like cheese?")
twiggy.log.name("second.child").fields(cheese="hate").warning("No")
twiggy.log.name("first").error("Can you do {0}", something)
twiggy.log.name("bob").debug("I wear pants")
try:
raise RuntimeError("Oh Noes!")
except RuntimeError:
twiggy.log.trace().critical("Went boom")
print("****************** all_ ***************************")
print(all_.stream.getvalue(), end=' ')
print("****************** out 1 **************************")
print(out1.stream.getvalue(), end=' ')
print("****************** out 2 **************************")
print(out2.stream.getvalue(), end=' ')
print("***************************************************")
# XXX this should really be done with a regex, but I'm feeling lazy
assert out1.stream.getvalue().startswith(
"""2010-10-28T02:15:57Z:INFO:second|do you like cheese?
2010-10-28T02:15:57Z:WARNING:second.child:cheese=hate|No
2010-10-28T02:15:57Z:ERROR:first|Can you do something cool
2010-10-28T02:15:57Z:DEBUG:bob|I wear pants
2010-10-28T02:15:57Z:CRITICAL|Went boom
TRACE Traceback (most recent call last):
""")
exception_line_re = re.compile(
r'TRACE File "/[^"]*/tests/test_integration.py",'
r' line [0-9]+,')
assert exception_line_re.search(out1.stream.getvalue())
assert out1.stream.getvalue().endswith(
"""TRACE raise RuntimeError("Oh Noes!")
TRACE RuntimeError: Oh Noes!
""")
assert out2.stream.getvalue() == \
"""2010-10-28T02:15:57Z:WARNING:second.child:cheese=hate|No
def test_integration(self):
everything = twiggy.outputs.StreamOutput(stream=StringIO(), format=twiggy.formats.line_format)
out1 = twiggy.outputs.StreamOutput(stream=StringIO(), format=twiggy.formats.line_format)
out2 = twiggy.outputs.StreamOutput(stream=StringIO(), format=twiggy.formats.line_format)
twiggy.add_emitters(
("*", twiggy.levels.DEBUG, None, everything),
("first", twiggy.levels.INFO, None, out1),
("second", twiggy.levels.DEBUG, twiggy.filters.glob_names("second.*"), out2),
("first-filter", twiggy.levels.DEBUG, ".*pants.*", out1),
)
def something():
return "something cool"
twiggy.log.debug("oh hi")
twiggy.log.name("second").info("do you like cheese?")
twiggy.log.name("second.child").fields(cheese="hate").warning("No")
twiggy.log.name("first").error("Can you do {0}", something)
twiggy.log.name("bob").debug("I wear pants")
try:
raise RuntimeError("Oh Noes!")
except:
twiggy.log.trace().critical("Went boom")
print("***************** everything **********************")
print(everything.stream.getvalue(), end=" ")
print("****************** out 1 **************************")
print(out1.stream.getvalue(), end=" ")
print("****************** out 2 **************************")
print(out2.stream.getvalue(), end=" ")
print("***************************************************")
# XXX this should really be done with a regex, but I'm feeling lazy
assert out1.stream.getvalue().startswith(
"""2010-10-28T02:15:57Z:INFO:second|do you like cheese?
2010-10-28T02:15:57Z:WARNING:second.child:cheese=hate|No
2010-10-28T02:15:57Z:ERROR:first|Can you do something cool
2010-10-28T02:15:57Z:DEBUG:bob|I wear pants
2010-10-28T02:15:57Z:CRITICAL|Went boom
TRACE Traceback (most recent call last):
"""
)
# """TRACE File "/home/pfein/Projects/python-twiggy/tests/test_integration.py", line 39, in test_integration
assert out1.stream.getvalue().endswith(
"""TRACE raise RuntimeError("Oh Noes!")
TRACE RuntimeError: Oh Noes!
"""
)
assert (
out2.stream.getvalue()
== """2010-10-28T02:15:57Z:WARNING:second.child:cheese=hate|No
"""
)
def test_format(self):
from twiggy import add_emitters
from twiggy import log
from twiggy.logging_compat import LoggingBridgeFormat, DEBUG, ERROR
from twiggy.outputs import ListOutput
logger = log.name("spam")
list_output = ListOutput(format=LoggingBridgeFormat())
messages = list_output.messages
add_emitters(("spam", DEBUG, None, list_output))
logger.error("eggs")
self.failUnlessEqual(messages[0], ('|eggs\n', ERROR, 'spam'))
def test_add_emitters(twiggy_output):
def myfilt(msg):
return True
twiggy.add_emitters(('test', twiggy.levels.INFO, myfilt, twiggy_output))
assert len(twiggy.emitters) == 1
e = twiggy.emitters['test']
assert isinstance(e, twiggy.filters.Emitter)
assert e.min_level == twiggy.levels.INFO
assert e.filter is myfilt
assert e._output is twiggy_output
def test_add_emitters(twiggy_output):
def myfilt(msg):
return True
twiggy.add_emitters(('test', twiggy.levels.INFO, myfilt, twiggy_output))
assert len(twiggy.emitters) == 1
e = twiggy.emitters['test']
assert isinstance(e, twiggy.filters.Emitter)
assert e.min_level == twiggy.levels.INFO
assert e.filter is myfilt
assert e._output is twiggy_output
def test_integration(self):
all_ = twiggy.outputs.StreamOutput(stream=StringIO(), format=twiggy.formats.line_format)
out1 = twiggy.outputs.StreamOutput(stream=StringIO(), format=twiggy.formats.line_format)
out2 = twiggy.outputs.StreamOutput(stream=StringIO(), format=twiggy.formats.line_format)
twiggy.add_emitters(('*', twiggy.levels.DEBUG, None, all_),
('first', twiggy.levels.INFO, None, out1),
('second', twiggy.levels.DEBUG,
twiggy.filters.glob_names('second.*'), out2),
('first-filter', twiggy.levels.DEBUG, ".*pants.*", out1))
def something():
return "something cool"
twiggy.log.debug("oh hi")
twiggy.log.name("second").info("do you like cheese?")
twiggy.log.name("second.child").fields(cheese="hate").warning("No")
twiggy.log.name("first").error("Can you do {0}", something)
twiggy.log.name("bob").debug("I wear pants")
try:
raise RuntimeError("Oh Noes!")
except RuntimeError:
twiggy.log.trace().critical("Went boom")
print("****************** all_ ***************************")
print(all_.stream.getvalue(), end=' ')
print("****************** out 1 **************************")
print(out1.stream.getvalue(), end=' ')
print("****************** out 2 **************************")
print(out2.stream.getvalue(), end=' ')
print("***************************************************")
# XXX this should really be done with a regex, but I'm feeling lazy
assert out1.stream.getvalue().startswith(
"""2010-10-28T02:15:57Z:INFO:second|do you like cheese?
2010-10-28T02:15:57Z:WARNING:second.child:cheese=hate|No
2010-10-28T02:15:57Z:ERROR:first|Can you do something cool
2010-10-28T02:15:57Z:DEBUG:bob|I wear pants
2010-10-28T02:15:57Z:CRITICAL|Went boom
TRACE Traceback (most recent call last):
""")
exception_line_re = re.compile(r'TRACE File "/[^"]*/tests/test_integration.py",'
r' line [0-9]+,')
assert exception_line_re.search(out1.stream.getvalue())
assert out1.stream.getvalue().endswith(
"""TRACE raise RuntimeError("Oh Noes!")
TRACE RuntimeError: Oh Noes!
""")
assert out2.stream.getvalue() == \
"""2010-10-28T02:15:57Z:WARNING:second.child:cheese=hate|No
def test_add_emitters(self):
out = twiggy.outputs.ListOutput(close_atexit = False)
def cleanup(out):
out.close()
self.addCleanup(cleanup, out)
def myfilt(msg):
return True
twiggy.add_emitters(('test', twiggy.levels.INFO, myfilt, out))
assert len(twiggy.emitters) == 1
e = twiggy.emitters['test']
assert isinstance(e, twiggy.filters.Emitter)
assert e.min_level == twiggy.levels.INFO
assert e.filter is myfilt
assert e._output is out
def test_add_emitters(self):
out = twiggy.outputs.ListOutput(close_atexit=False)
def cleanup(out):
out.close()
self.addCleanup(cleanup, out)
def myfilt(msg):
return True
twiggy.add_emitters(('test', twiggy.levels.INFO, myfilt, out))
assert len(twiggy.emitters) == 1
e = twiggy.emitters['test']
assert isinstance(e, twiggy.filters.Emitter)
assert e.min_level == twiggy.levels.INFO
assert e.filter is myfilt
assert e._output is out
def setup_emitters(twiggy_output):
def myfilt(msg):
return True
twiggy.add_emitters(('test', twiggy.levels.INFO, myfilt, twiggy_output))
twiggy.add_emitters(('second', twiggy.levels.INFO, myfilt, twiggy_output))
twiggy.add_emitters(('*', twiggy.levels.INFO, myfilt, twiggy_output))
def setup_emitters(twiggy_output):
def myfilt(msg):
return True
twiggy.add_emitters(('test', twiggy.levels.INFO, myfilt, twiggy_output))
twiggy.add_emitters(('second', twiggy.levels.INFO, myfilt, twiggy_output))
twiggy.add_emitters(('*', twiggy.levels.INFO, myfilt, twiggy_output))
def set_log_level(level=levels.INFO):
emitters.clear()
add_emitters(('*', level, None, diesel_output))
# If any of the emitters contain MPIOutput instances, they need to be replaced
# by newly initialized instances so that they write to valid file handles and
# use the intercommunicator to the parent process:
for k, v in emitters.iteritems():
if isinstance(v._output, neurokernel.tools.mpi.MPIOutput):
level = v.min_level
name = v._output.filename
format = v._output._format
mode = v._output.mode
# The close_atexit argument is explicitly set to False here because we need
# to manually close the file handle associated with MPIOutput before
# MPI.Finalize() is called via atexit in the base/core modules:
twiggy.add_emitters(('file', level, None,
neurokernel.tools.mpi.MPIOutput(name, format,
MPI.COMM_WORLD, mode, False)))
else:
twiggy.emitters[k] = v
# Get the routing table:
routing_table = parent.bcast(None, root=0)
# Get the target class/function and its constructor arguments:
target, target_globals, kwargs = parent.recv()
# Insert the transmitted globals into the current scope:
globals()[target.__name__] = target
for k, n in target_globals.iteritems():
globals()[k] = n
def setUp(self):
self.logger = getLogger("spam")
self.logger.setLevel(DEBUG)
self.list_output = ListOutput()
self.messages = self.list_output.messages
add_emitters(("spam", DEBUG, None, self.list_output))
def test_sanity(self):
logger = log.name("decoy")
add_emitters(("decoy", DEBUG, None, LoggingBridgeOutput()))
logger.error("spam")
logger.notice("eggs")
def setup_logger(name='', level=twiggy.levels.DEBUG,
fmt=twiggy.formats.line_format,
fmt_name=('{0:%s}' % 10).format,
screen=None, file_name=None,
mpi_comm=None, zmq_addr=None,
log_exceptions=True, multiline=False):
"""
Setup a twiggy logger.
Parameters
----------
name : str
Logger name.
level : twiggy.levels.LogLevel
Logging level.
fmt : twiggy.formats.LineFormat
Logging formatter class instance.
fmt_name : function
Function with one parameter that formats the message name.
screen : bool
Create output stream handler to the screen if True.
file_name : str
Create output handler to specified file.
mpi_comm : mpi4py.MPI.Intracomm
If not None, use MPI I/O with the specified communicator for
output file handler. Ignored if the `file_name` parameter
is not specified.
zmq_addr : str
ZeroMQ socket address.
log_exceptions : bool
If True, exception messages are written to the logger.
multiline : bool
If True, log exception messages on multiple lines.
Returns
-------
logger : twiggy.logger.Logger
Configured logger.
Bug
---
To use the ZeroMQ output class with multiprocessing, it must be added
as an emitter within each process.
"""
fmt = copy.copy(fmt)
fmt.conversion.delete('name')
# Apply name format to the value (i.e., the name), not the key (i.e., the
# field name "name"):
fmt.conversion.add('name', str, lambda k, v: fmt_name(v))
if file_name:
if mpi_comm:
if 'mpi4py.MPI' not in sys.modules:
raise ValueError('mpi4py not available')
if not isinstance(mpi_comm, mpi4py.MPI.Intracomm):
raise ValueError('mpi_comm must be an instance of '
'mpi4py.MPI.Intracomm')
if 'neurokernel.tools.mpi' not in sys.modules:
raise ValueError('neurokernel.tools.mpi not available')
file_output = \
neurokernel.tools.mpi.MPIOutput(file_name, fmt, mpi_comm)
else:
file_output = \
twiggy.outputs.FileOutput(file_name, fmt, 'w')
twiggy.add_emitters(('file', level, None, file_output))
if screen:
screen_output = \
twiggy.outputs.StreamOutput(fmt, stream=sys.stdout)
twiggy.add_emitters(('screen', level, None, screen_output))
if zmq_addr:
if 'neurokernel.tools.zmq' not in sys.modules:
raise ValueError('neurokernel.tools.zmq not available')
zmq_output = neurokernel.tools.zmq.ZMQOutput(zmq_addr, fmt)
twiggy.add_emitters(('zmq', level, None, zmq_output))
logger = twiggy.log.name(fmt_name(name))
if log_exceptions:
set_excepthook(logger, multiline)
return logger
def log_setup(level, stream=sys.stderr):
output = twiggy.outputs.StreamOutput(twiggy.formats.line_format,
stream=stream)
twiggy.add_emitters(('*', twiggy.levels.name2level(level), True, output))
def twiggy_setup():
sout = outputs.StreamOutput(format=formats.line_format)
add_emitters(('ipborg.std', levels.DEBUG, None, sout))
def setup_logger(name='',
level=twiggy.levels.DEBUG,
fmt=twiggy.formats.line_format,
fmt_name=('{0:%s}' % 10).format,
screen=None,
file_name=None,
mpi_comm=None,
zmq_addr=None,
log_exceptions=True,
multiline=False):
"""
Setup a twiggy logger.
Parameters
----------
name : str
Logger name.
level : twiggy.levels.LogLevel
Logging level.
fmt : twiggy.formats.LineFormat
Logging formatter class instance.
fmt_name : function
Function with one parameter that formats the message name.
screen : bool
Create output stream handler to the screen if True.
file_name : str
Create output handler to specified file.
mpi_comm : mpi4py.MPI.Intracomm
If not None, use MPI I/O with the specified communicator for
output file handler. Ignored if the `file_name` parameter
is not specified.
zmq_addr : str
ZeroMQ socket address.
log_exceptions : bool
If True, exception messages are written to the logger.
multiline : bool
If True, log exception messages on multiple lines.
Returns
-------
logger : twiggy.logger.Logger
Configured logger.
Bug
---
To use the ZeroMQ output class with multiprocessing, it must be added
as an emitter within each process.
"""
fmt = copy.copy(fmt)
fmt.conversion.delete('name')
# Apply name format to the value (i.e., the name), not the key (i.e., the
# field name "name"):
fmt.conversion.add('name', str, lambda k, v: fmt_name(v))
if file_name:
if mpi_comm:
if 'mpi4py.MPI' not in sys.modules:
raise ValueError('mpi4py not available')
if not isinstance(mpi_comm, mpi4py.MPI.Intracomm):
raise ValueError('mpi_comm must be an instance of '
'mpi4py.MPI.Intracomm')
if 'neurokernel.tools.mpi' not in sys.modules:
raise ValueError('neurokernel.tools.mpi not available')
file_output = \
neurokernel.tools.mpi.MPIOutput(file_name, fmt, mpi_comm)
else:
file_output = \
twiggy.outputs.FileOutput(file_name, fmt, 'w')
twiggy.add_emitters(('file', level, None, file_output))
if screen:
screen_output = \
twiggy.outputs.StreamOutput(fmt, stream=sys.stdout)
twiggy.add_emitters(('screen', level, None, screen_output))
if zmq_addr:
if 'neurokernel.tools.zmq' not in sys.modules:
raise ValueError('neurokernel.tools.zmq not available')
zmq_output = neurokernel.tools.zmq.ZMQOutput(zmq_addr, fmt)
twiggy.add_emitters(('zmq', level, None, zmq_output))
logger = twiggy.log.name(fmt_name(name))
if log_exceptions:
set_excepthook(logger, multiline)
return logger
def set_log_level(level=levels.INFO):
emitters.clear()
add_emitters(
('*', level, None, diesel_output)
)
def setup_logger(name='', level=twiggy.levels.DEBUG,
fmt=twiggy.formats.line_format,
fmt_name=('{0:%s}' % 10).format,
screen=None, file_name=None, zmq_addr=None,
log_exceptions=True, multiline=False):
"""
Setup a twiggy logger.
Parameters
----------
name : str
Logger name.
level : twiggy.levels.LogLevel
Logging level.
fmt : twiggy.formats.LineFormat
Logging formatter class instance.
fmt_name : function
Function with one parameter that formats the message name.
screen : bool
Create output stream handler to the screen if True.
file_name : str
Create output handler to specified file.
zmq_addr : str
ZeroMQ socket address.
log_exceptions : bool
If True, exception messages are written to the logger.
multiline : bool
If True, log exception messages on multiple lines.
Returns
-------
logger : twiggy.logger.Logger
Configured logger.
Bug
---
To use the ZeroMQ output class with multiprocessing, it must be added
as an emitter within each process.
"""
fmt = copy.copy(fmt)
fmt.conversion.delete('name')
# Apply name format to the value (i.e., the name), not the key (i.e., the
# field name "name"):
fmt.conversion.add('name', str, lambda k, v: fmt_name(v))
if file_name:
file_output = \
twiggy.outputs.FileOutput(file_name, fmt, 'w')
twiggy.add_emitters(('file', level, None, file_output))
if screen:
screen_output = \
twiggy.outputs.StreamOutput(fmt, stream=sys.stdout)
twiggy.add_emitters(('screen', level, None, screen_output))
if zmq_addr:
zmq_output = ZMQOutput(zmq_addr, fmt)
twiggy.add_emitters(('zmq', level, None, zmq_output))
logger = twiggy.log.name(fmt_name(name))
if log_exceptions:
set_excepthook(logger, multiline)
return logger