def duplicate_handlers(old_handlers, formatter):
log_files = []
new_handlers = []
for handler in old_handlers:
if isinstance(handler, logging.FileHandler):
log_name = handler.baseFilename
if log_name in log_files:
continue
log_files.append(log_name)
task_file_out_handler = logging.FileHandler(log_name)
task_file_out_handler.setFormatter(formatter)
task_file_out_handler.setLevel(handler.level)
new_handlers.append(task_file_out_handler)
if isinstance(handler, MemoryHandler) and isinstance(
handler.target, logging.FileHandler):
log_name = handler.target.baseFilename
if log_name in log_files:
continue
log_files.append(log_name)
task_file_out_handler = logging.FileHandler(log_name, delay=True)
task_file_out_handler.setFormatter(formatter)
task_file_out_handler.setLevel(handler.level)
task_mem_handler = MemoryHandler(handler.capacity,
flushLevel=handler.flushLevel)
task_mem_handler.setFormatter(formatter)
task_mem_handler.setLevel(handler.level)
task_mem_handler.setTarget(task_file_out_handler)
new_handlers.append(task_mem_handler)
return new_handlers
def full_context_error_logger():
""" capture all log specific to a context
:return:
"""
from logging.handlers import MemoryHandler
from StringIO import StringIO
buffer = StringIO()
logLevel = logging.DEBUG
streamhandler = logging.StreamHandler(buffer)
streamhandler.setLevel(logLevel)
streamhandler.setFormatter(formatter)
memory_handler = MemoryHandler(capacity=1024 * 100,
flushLevel=logging.ERROR,
target=streamhandler)
memory_handler.setLevel(logLevel)
memory_handler.setFormatter(formatter)
rootLogger = logging.getLogger()
rootLogger.addHandler(memory_handler)
result = {"error_log": None}
try:
yield result
except:
memory_handler.flush()
buffer.flush()
result["error_log"] = buffer.getvalue() + traceback.format_exc()
finally:
rootLogger.removeHandler(memory_handler)
memory_handler.close()
buffer.close()
def add_handler(handler):
"""
Add a new handler, wrapped in a MemoryHandler for buffering.
"""
# Use MemoryHandler as an intermediary for buffering. Essentially: we don't
# want to flush every message, since that presents a severe scalability
# issue.
mem_handler = MemoryHandler(32, target=handler)
mem_handler.setFormatter(logger_formatter)
# Real filtering will be done by the logger itself
mem_handler.setLevel(DEBUG)
MEM_HANDLERS.append(mem_handler)
base_logger.addHandler(mem_handler)
def create_logging_handler_for_collection(tempdir, prefix):
from sys import maxsize
from os import path
from logging import FileHandler, DEBUG, Formatter
from logging.handlers import MemoryHandler
target = FileHandler(path.join(tempdir, "{}.{}.debug.log".format(prefix, get_timestamp())))
target.setFormatter(Formatter(**LOGGING_FORMATTER_KWARGS))
handler = MemoryHandler(maxsize, target=target)
handler.setLevel(DEBUG)
try:
yield handler
finally:
handler.close()
def add_module_logger_to_schoollib():
global _module_handler
if _module_handler is None:
module_handler = ModuleHandler(udebug_facility=ud.MODULE)
_module_handler = MemoryHandler(-1,
flushLevel=logging.DEBUG,
target=module_handler)
_module_handler.setLevel(logging.DEBUG)
logger.addHandler(_module_handler)
else:
logger.info(
'add_module_logger_to_schoollib() should only be called once! Skipping...'
)
return _module_handler
def create_logging_handler_for_collection(tempdir, prefix):
from sys import maxsize
from os import path
from logging import FileHandler, DEBUG, Formatter
from logging.handlers import MemoryHandler
target = FileHandler(
path.join(tempdir, "collection-logs",
"{}.{}.debug.log".format(prefix, get_timestamp())))
target.setFormatter(Formatter(**LOGGING_FORMATTER_KWARGS))
handler = MemoryHandler(maxsize, target=target)
handler.setLevel(DEBUG)
try:
yield handler
finally:
handler.close()
target.close()
from logging.handlers import MemoryHandler
from IPython.core.magic import (Magics, magics_class, line_magic)
from collections import deque
logMessageBufferSize = 200
logMessages=deque([], logMessageBufferSize)
class PixiedDustLoggingHandler(logging.Handler):
def emit(self, record):
logMessages.append((record.levelno, self.format(record)))
#init pixiedust logging
pixiedustHandler = PixiedDustLoggingHandler()
pixiedustHandler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
memHandler = MemoryHandler(1, target=pixiedustHandler)
memHandler.setLevel(logging.DEBUG)
pixiedustLogger = logging.getLogger("PixieDust")
pixiedustLogger.addHandler(memHandler)
pixiedustLogger.setLevel(logging.DEBUG)
@magics_class
class PixiedustLoggingMagics(Magics):
@line_magic
def pixiedustLog(self, arg_s):
try:
opts,args = self.parse_options( arg_s, "l:f:m:")
level = logging.getLevelName( opts.get("l", "INFO").upper() )
if not isinstance(level, int):
level = logging.getLevelName("INFO")
lookup = opts.get("f", None)
else:
sys.stdout = sys.__stdout__ #old_stdout
sys.stderr = sys.__stderr__
#adds a stream catcher for display and a memory handler for saving
log_stream = StreamCatch()
logger = getLogger()
display_handler = StreamHandler(stream=log_stream)
display_handler.setLevel(LOGLEVEL)
display_handler.setFormatter(Formatter(LOGFORMATTER))
display_handler.name = "StreamCatch"
logger.addHandler(display_handler)
memory_handler = MemoryHandler(MEMBUFFER)
memory_handler.setLevel(LOGLEVEL)
memory_handler.setFormatter(Formatter(LOGFORMATTER))
memory_handler.name = "MemoryLog"
logger.addHandler(memory_handler)
log_info("Started logging")
def make_log_file(log_path, mode='a'):
"""Points memory handler at a particular file to save the log."""
file_handler = FileHandler(filename=log_path, mode=mode)
file_handler.setLevel(LOGLEVEL)
file_handler.setFormatter(Formatter(LOGFORMATTER))
memory_handler.setTarget(file_handler)
sys.stderr=self
else:
sys.stdout=sys.__stdout__ #old_stdout
sys.stderr=sys.__stderr__
#adds a stream catcher for display and a memory handler for saving
log_stream=StreamCatch()
logger=getLogger()
display_handler=StreamHandler(stream=log_stream)
display_handler.setLevel(LOGLEVEL)
display_handler.setFormatter(Formatter(LOGFORMATTER))
display_handler.name="StreamCatch"
logger.addHandler(display_handler)
memory_handler=MemoryHandler(MEMBUFFER)
memory_handler.setLevel(LOGLEVEL)
memory_handler.setFormatter(Formatter(LOGFORMATTER))
memory_handler.name="MemoryLog"
logger.addHandler(memory_handler)
log_debug("Started logging")
def make_log_file(log_path, mode='a'):
"""Points memory handler at a particular file to save the log."""
file_handler = FileHandler(filename=log_path, mode=mode)
file_handler.setLevel(LOGLEVEL)
file_handler.setFormatter(Formatter(LOGFORMATTER))
memory_handler.setTarget(file_handler)
def remove_log_file():
"""closes the log file and removes memory_handler from pointing at it"""
import os
import os.path
import pkg_resources
from pyphant.core.Helpers import getPyphantPath
LOGDIR = getPyphantPath()
import logging
from logging.handlers import MemoryHandler
logging.basicConfig(level=logging.NOTSET,
filename=os.path.join(LOGDIR, u'pyphant.log'),
filemode='w',
format="%(asctime)s - %(levelname)s:%(name)s:%(thread)"\
"d:%(module)s.%(funcName)s(l %(lineno)d):%(message)s")
console = logging.StreamHandler()
console.setLevel(logging.WARNING)
pdmh = MemoryHandler(1000, flushLevel=logging.CRITICAL + 1)
pdmh.setLevel(logging.WARNING)
logging.getLogger('').addHandler(pdmh)
logging.getLogger('').addHandler(console)
import sys
import wx
import wx.aui
import sogl
import pyphant.wxgui2.paramvisualization.ParamVisReg as ParamVisReg
from pyphant.core.H5FileHandler import H5FileHandler
from pyphant.wxgui2 import WorkerRepository
from pyphant.wxgui2 import ConfigureFrame
import platform
from pyphant.core.KnowledgeManager import KnowledgeManager
import webbrowser
pltform = platform.system()
# Original punctuation should be out of the brackets
self.positive("I am hear!", "I am [here]!")
# Drop leading and trailing spaces
self.positive(" their is ", "[there] is")
# Retain double spaces
self.positive("their is", "[there] is")
# Drop periods, commas, and semicolons
self.positive("their is.", "[there] is")
self.positive("their is,", "[there] is")
self.positive("their is;", "[there] is")
def test_wording(self):
"""Verify that wording can be generated without failing"""
self.positive(
"Their is and your don't supposed to! (blah) They think their is.")
logging.debug(self.parser.generate_wording('@@')
.encode('ascii', 'replace'))
# not implemented yet
self.assertRaises(NotImplementedError,
self.parser.generate_wording_long, '')
if __name__ == '__main__':
stream_null = StringIO()
logging.basicConfig(stream=stream_null, level=logging.DEBUG)
handler_stream = StreamHandler(stream=sys.stderr)
handler_mem = MemoryHandler(1024, target=handler_stream)
handler_mem.setLevel(logging.DEBUG)
handler_mem.setFormatter(logging.Formatter())
logging.getLogger().addHandler(handler_mem)
unittest.main()
import os, os.path, pkg_resources
from pyphant.core.Helpers import getPyphantPath
LOGDIR = getPyphantPath()
import logging
from logging.handlers import MemoryHandler
logging.basicConfig(level=logging.NOTSET,
filename=os.path.join(LOGDIR, u'pyphant.log'),
filemode='w',
format="%(asctime)s - %(levelname)s:%(name)s:%(thread)"\
"d:%(module)s.%(funcName)s(l %(lineno)d):%(message)s")
console = logging.StreamHandler()
console.setLevel(logging.WARNING)
pdmh = MemoryHandler(1000, flushLevel=logging.CRITICAL + 1)
pdmh.setLevel(logging.WARNING)
logging.getLogger('').addHandler(pdmh)
logging.getLogger('').addHandler(console)
import sys
import wx
import wx.aui
import sogl
import pyphant.wxgui2.paramvisualization.ParamVisReg as ParamVisReg
from pyphant.core.H5FileHandler import H5FileHandler
import WorkerRepository
import ConfigureFrame
import platform
from pyphant.core.KnowledgeManager import KnowledgeManager
import webbrowser
from logging.handlers import MemoryHandler
from IPython.core.magic import (Magics, magics_class, line_magic)
from collections import deque
logMessageBufferSize = 200
logMessages=deque([], logMessageBufferSize)
class PixiedDustLoggingHandler(logging.Handler):
def emit(self, record):
logMessages.append((record.levelno, self.format(record)))
#init pixiedust logging
pixiedustHandler = PixiedDustLoggingHandler()
pixiedustHandler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
memHandler = MemoryHandler(1, target=pixiedustHandler)
memHandler.setLevel(logging.DEBUG)
pixiedustLogger = logging.getLogger("PixieDust")
pixiedustLogger.addHandler(memHandler)
pixiedustLogger.setLevel(logging.DEBUG)
@magics_class
class PixiedustLoggingMagics(Magics):
@line_magic
def pixiedustLog(self, arg_s):
try:
opts,args = self.parse_options( arg_s, "l:f:m:")
level = logging.getLevelName( opts.get("l", "INFO").upper() )
if not isinstance(level, int):
level = logging.getLevelName("INFO")
lookup = opts.get("f", None)
