def _emit(self, record): # Following snippett taken from WatchedFileHandler in Python 2.6 # Reopen the stream if the original file has been moved if not os.path.exists(self.baseFilename): stat = None changed = 1 else: stat = os.stat(self.baseFilename) changed = (stat[ST_DEV] != self.dev) or (stat[ST_INO] != self.ino) if changed and self.stream is not None: try: self.stream.flush() except: pass try: self.stream.close() except: pass self.stream = self._reopen() if stat is None: stat = os.stat(self.baseFilename) self.dev, self.ino = stat[ST_DEV], stat[ST_INO] # Now that we've reopened the file if necessary, call the regular # emit() routine for the rotating handler. RotatingFileHandler.emit(self, record)
class MultiProcessingFileHandler(LogHandler): def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None): LogHandler.__init__(self) self._handler = RotatingFileHandler(filename, mode, maxBytes, backupCount, encoding) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): LogHandler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) # print('received on pid {}'.format(os.getpid())) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args have been stringified. # Removes any chance of unpickleable things inside and # possibly reduces message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() LogHandler.close(self)
class MultiProcessingLog(logging.Handler): def __init__(self, filename, mode="a", maxBytes=0, backupCount=0, encoding=None, delay=0): logging.Handler.__init__(self) # In case our call to RotatingFileHandler blows up we first set # the _handler to None self._handler = None self._handler = RotatingFileHandler(filename, mode, maxBytes, backupCount, encoding, delay) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): if self._handler is not None: self._handler.close() logging.Handler.close(self)
class MultiProcessingLog(logging.Handler): def __init__(self, filename, mode="timed", max_bytes=0, backup_count=0): logging.Handler.__init__(self) self.queue = multiprocessing.Queue(-1) if mode == "timed": self._handler = TimedRotatingFileHandler(filename, "midnight", 1, backup_count) else: self._handler = RotatingFileHandler(filename, "a", max_bytes, backup_count) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while self.queue.empty() == False: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
class MultiProcessingLog(logging.Handler): def __init__(self, name, mode = 'a', maxsize = 0, rotate = 0): ''' By default the RotatingFileHandler is set as an usual Handler (no rotating, no maxsize) ''' logging.Handler.__init__(self) self._handler = RotatingFileHandler(name, mode, maxsize, rotate) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
class MultiProcessingLog(logging.Handler): def __init__(self, name, mode='a', maxsize=0, rotate=0): ''' By default the RotatingFileHandler is set as an usual Handler (no rotating, no maxsize) ''' logging.Handler.__init__(self) self._handler = RotatingFileHandler(name, mode, maxsize, rotate) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
def emit(self, record): try: logrecord = logReCord(record) sys.tracebacklimit = 1 RotatingFileHandler.emit(self, logrecord) except (KeyboardInterrupt, SystemExit): raise except Exception: self.handleError(record)
class MultiProcessingFileHandler(LogHandler): def __init__(self, filename, mode='a', max_bytes=0, backup_count=0, encoding=None): LogHandler.__init__(self) self._handler = RotatingFileHandler(filename, mode, max_bytes, backup_count, encoding) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): LogHandler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: pass traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() LogHandler.close(self)
def emit(self, record: object) -> None: """Emit a record. Emit logging events only if handler_key matches thread_key. Args: record: The record to be logged. """ # handler_key and thread_key are added in logger.add_thread_file_handler() method if hasattr(threading.current_thread(), self.thread_key): if self.handler_key == getattr(threading.current_thread(), self.thread_key): RotatingFileHandler.emit(self, record)
class BackgroundRotatingFileHandler(BackgroundLoggingHandler): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) #TODO: rewrite RotatingFileHandler to not call format twice (once on roolover check and once on emit) self.rotating_file_handler = RotatingFileHandler(*args, **kwargs) def setFormatter(self, formatter): self.formatter = formatter self.rotating_file_handler.formatter = formatter def process_record(self, record): self.rotating_file_handler.emit(record)
class MultiprocessingRotatingFileHandler(logging.Handler): def __init__(self, *args, **kwargs): logging.Handler.__init__(self) self._handler = RotatingFileHandler(*args, **kwargs) self.queue = multiprocessing.Queue() t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
def do_emit(self, record): if self.last_record and \ record.msg == self.last_record.msg and \ record.lineno == self.last_record.lineno and \ record.pathname == self.last_record.pathname: self.count_repeat += 1 else: if self.count_repeat: self.last_record.msg = 'Last line repeats %s times' % self.count_repeat RotatingFileHandler.emit(self, self.last_record) self.count_repeat = 0 self.last_record = record RotatingFileHandler.emit(self, record)
class SimpleLog(logging.Handler): def __init__(self, filename, mode='sized', max_bytes=0, backup_count=0): logging.Handler.__init__(self) if mode == 'timed': self._handler = TimedRotatingFileHandler(filename, 'midnight', 1, backup_count) else: self._handler = RotatingFileHandler(filename, 'a', max_bytes, backup_count) def emit(self, record): self._handler.emit(record) def setFormatter(self, fmt): self._handler.setFormatter(fmt) def close(self): self._handler.close()
class SmoresFileLog(logging.Handler): def __init__(self, name, mode, maxsize=0, backup=1): global handler_list logging.Handler.__init__(self) self._handler = RotatingFileHandler(name, mode, maxBytes=maxsize, backupCount=backup, encoding='UTF-8') if self.name not in handler_list: handler_list.append(name) try: self._handler.doRollover() except (PermissionError, ValueError): pass def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def _format_record(self, record): # ensure that exc_info and args have been stringified. Removes any # chance of unpickleable things inside and possibly reduces message size # sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self._handler.emit(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
def emit(self, record): if self._enabled: RotatingFileHandler.emit(self, record)
class MultiProcessingLog(logging.Handler): """ Synchronised parallel multi processing log. """ def __init__(self, name, mode, maxsize, rotate, encoding="utf-8", delay=0): logging.Handler.__init__(self) self._handler = RotatingFileHandler(name, mode, maxsize, rotate, encoding, delay) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): """ Default setFormatter impl. """ logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): """ Receives one record to log. """ while True: try: record = self.queue.get() if self._handler: self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): """ Puts to nowait queue. """ self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: # noinspection PyUnusedLocal dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() self._handler = None logging.Handler.close(self)
def emit(self, record): self.doRollover() RotatingFileHandler.emit(self, record)
class Log(logging.Handler): ''' Queue-based single-file logger for parallel processes Args: name (str): name of the logger mode (str): write mode; defaults to 'a' (append) maxsize (int): maximum size in bytes of log file rotate (int): number of historical log files to retain; defaults to 0 ''' def __init__(self, name, mode, maxsize, rotate): # type: (LogType, str, str, int, int) -> None logging.Handler.__init__(self) self._handler = RotatingFileHandler(name, mode, maxsize, rotate) self.queue = multiprocessing.Queue(-1) #type: multiprocessing.Queue t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): # type: (LogType, logging.Formatter) -> None ''' Set logging formatter Args: fmt (:py:class:`logging.Formatter`): valid formatter; read more about valid formatters `here <https://docs.python.org/3/library/logging.html#formatter-objects>`_ ''' logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): # type: (LogType) -> None ''' Receive logging records from queue Raises: Exception: re-raise after :py:class:`KeyboardInterrupt` or :py:class:`SystemExit` ''' while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): # type: (LogType, LogRecord) -> None ''' Send logging record to logging queue Args: s (:py:class`logging.LogRecord`): record to be logged ''' self.queue.put_nowait(s) def _format_record(self, record): # type: (LogType, logging.LogRecord) -> logging.LogRecord ''' Format record to ensure it is pickle-able Args: record (:py:class:`logging.LogRecord`): logging record ''' # ensure that exc_info and args have been stringified. Removes any # chance of unpickleable things inside and possibly reduces message size # sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): # type: (LogType, logging.LogRecord) -> None ''' Format and send record to queue Args: record (:py:class:`logging.LogRecord`): logging record Raises: Exception: re-raise after :py:class:`KeyboardInterrupt` or :py:class:`SystemExit` ''' try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): # type: (LogType) -> None ''' Close logger and all associated handlers ''' self._handler.close() logging.Handler.close(self)
class MultiProcessingLog(logging.Handler): def __init__(self, filename, mode='timed', max_bytes=0, backup_count=0): logging.Handler.__init__(self) self.queue = multiprocessing.Queue(-1) if mode == 'timed': self._handler = TimedRotatingFileHandler(filename, 'midnight', 1, backup_count) else: self._handler = RotatingFileHandler(filename, 'a', max_bytes, backup_count) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while self.queue.empty() == False: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
def emit(x, record): RotatingFileHandler.emit(x, record) if not app.debug else None
class MultiprocessingLogHandler(logging.Handler): """ by zzzeek on SO: http://stackoverflow.com/questions/641420/how-should-i-log-while-using-multiprocessing-in-python """ def __init__(self, *args, **kwargs): logging.Handler.__init__(self) self.lock = None if 'klass' in kwargs: klass = kwargs['klass'] del kwargs['klass'] self._handler = klass(*args, **kwargs) else: self._handler = RotatingFileHandler(*args, **kwargs) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
def emit(self, record): if not self.__doneInitialRollover: self.doRollover() self.__doneInitialRollover = True return RotatingFileHandler.emit(self, record)
def emit(self, record): try: self._rotlock.acquire(True) RotatingFileHandler.emit(self, record) finally: self._rotlock.release()
class TemperatureLogger(): """Starts recording temperatures from two TMP100 chips attached to the I2C bus. Resolution determined by BITS. Sensors read every LOG_INTERVAL, recording to TEMPERATURE_CSV_FILE. """ semaphore = Semaphore(blocking=False) log_timer = Timer(offset=1, interval=LOG_INTERVAL) def __init__(self): self._terminated = False self.timestamp = None self.temperatures = [None, None] self.log_handler = RotatingFileHandler(TEMPERATURE_CSV_FILE, backupCount=10) self.log_handler.setFormatter(TempFormatter()) def run(self): logging.info("Temperature logging starting...") with i2c.I2CMaster() as bus,\ Selector() as selector,\ self.semaphore,\ self.log_timer,\ Timer(offset=0.1, interval=LCD_INTERVAL) as lcd_timer,\ Adafruit_LCDPlate.get_interrupt_pin() as int_pin: # tolerate missing LCD lcd = None try: lcd = Adafruit_LCDPlate.Adafruit_LCDPlate(bus) fmt = FMT_ALL logging.info("LCD is attached") except IOError: logging.info("LCD is not attached") # resolution (9-12 bit) gets stored as (0-3) in bits 5,6 of config reg config = (BITS - 9) << 5 bus.transaction( i2c.writing_bytes(TMP100A_ADDRESS, CONFIG_REGISTER, config), i2c.writing_bytes(TMP100B_ADDRESS, CONFIG_REGISTER, config)) for addr in (TMP100A_ADDRESS, TMP100B_ADDRESS): read_results = bus.transaction( i2c.writing_bytes(addr, CONFIG_REGISTER), i2c.reading(addr, 1)) # should be 0x80 at startup/reset logging.info("conf register for 0x{0:02x}: 0x{1:02x}".format(addr, read_results[0][0])) right_shift = 16 - BITS div = 1 << (BITS - 8) selector.add(self.semaphore) selector.add(self.log_timer) self.log_timer.start() if lcd: selector.add(lcd_timer) selector.add(int_pin) lcd_timer.start() while not self._terminated: try: if lcd: lcd.set_led(Adafruit_LCDPlate.GREEN_PIN, False) selector.wait() if self._terminated: break if not selector.ready is int_pin: selector.ready.wait() # consume event for i, addr in enumerate((TMP100A_ADDRESS, TMP100B_ADDRESS)): read_results = bus.transaction( i2c.writing_bytes(addr, TEMP_REGISTER), i2c.reading(addr, 2)) t = read_results[0] self.temperatures[i] = (t[0] if t[0] <= 127 else t[0] - 256) + (t[1] >> right_shift) / div self.timestamp = time() if lcd: lcd.set_led(Adafruit_LCDPlate.GREEN_PIN, True) lcd.set_led(Adafruit_LCDPlate.RED_PIN, False) if selector.ready is self.log_timer: self.log_handler.emit(TempLogRecord(self.timestamp, self.temperatures)) elif selector.ready is lcd_timer: lcd.clear() lcd.message(fmt.format( self.temperatures[0], c2f(self.temperatures[0]), self.temperatures[1], c2f(self.temperatures[1]))) elif selector.ready is int_pin: # this is hacky... intcap = lcd.chip.registers.read_register(INTCAPA) # required? button_state = lcd.get_button_state() if button_state == 0: pass # button up event: ignore else: if lcd.button_state_pressed(button_state, Adafruit_LCDPlate.SELECT): fmt = FMT_ALL elif lcd.button_state_pressed(button_state, Adafruit_LCDPlate.UP): fmt = FMT_INSIDE elif lcd.button_state_pressed(button_state, Adafruit_LCDPlate.DOWN): fmt = FMT_OUTSIDE elif lcd.button_state_pressed(button_state, Adafruit_LCDPlate.LEFT): fmt = FMT_C elif lcd.button_state_pressed(button_state, Adafruit_LCDPlate.RIGHT): fmt = FMT_F else: fmt = "Unknown\nButton!" lcd_timer.stop() lcd_timer.start() except KeyboardInterrupt: logging.info("Keyboard interrupt received") self._terminated = True except SystemExit: logging.info("SystemExit received") self._terminated = True except: logging.exception("Error reading sensors:") if lcd: lcd.set_led(Adafruit_LCDPlate.RED_PIN, True) if lcd: lcd.clear() lcd.set_backlight(False) lcd.set_led(Adafruit_LCDPlate.GREEN_PIN, False) lcd.set_led(Adafruit_LCDPlate.RED_PIN, False) logging.info("Temperature logging stopped.") self.log_handler.close() def terminate(self): self._terminated = True self.semaphore.signal() def rotate_log(self): self.log_timer.stop() self.log_handler.doRollover() self.log_timer.start()
class QueuedHandler(logging.Handler): def __init__(self, *args, **kwargs): logging.Handler.__init__(self) self._handler = RotatingFileHandler(*args, **kwargs) # self._handler.setLevel(logging.DEBUG) self.queue = multiprocessing.Queue(-1) self.t = threading.Thread(target=self.receive) self.t.daemon = True self.t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() if record is None: break self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args have been stringified. Removes any # chance of unpickleable things inside and possibly reduces message size # sent over the pipe record.msg = str(os.getppid()) + "_" + str(id(self)) + "_" + record.msg if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self.send(None) self.t.join() # wait for queue to empty before closing self._handler.close() logging.Handler.close(self)
class CustomLogHandler(logging.Handler, metaclass=Singleton): """ multiprocessing log handler This handler makes it possible for several processes to log to the same file by using a queue. """ def __init__(self, fname): logging.Handler.__init__(self) # self._handler = FH(fname) # Change log file after every 200MB of data pushed to the file self._handler = RotatingFileHandler(fname, maxBytes=200000000, backupCount=10) self.queue = multiprocessing.Queue(-1) thrd = threading.Thread(target=self.receive) thrd.daemon = True thrd.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
class RotatingFile(Thread): """ A class that acts like the logging.handlers.RotatingFileHander; writing data out to a file until file grows too large, then rolling over. This class is intended to be used for stdout and stderr for calls to subprocess module routines, such as the Popen constructor. However, there are many other applications for this class. """ def __init__(self, *args, **kwargs): super().__init__() self.rw = None # Initialize read/write pipe as None self.callback = kwargs.pop( 'callback', None) # Pop off 'callback' keyword; set to None for key, val in ROTATING_FORMAT.items(): if key not in kwargs: kwargs[key] = val formatter = kwargs.pop('formatter', None) self.log = RotatingFileHandler( *args, **kwargs) # Initialize rotating file handler if isinstance(formatter, logging.Formatter): self.log.setFormatter(formatter) # Set formatting else: self.log.setFormatter( logging.Formatter('%(asctime)s - %(message)s')) def __enter__(self, *args, **kwargs): self.start() def __exit__(self, *args, **kwargs): self.close() def setFormatter(self, *args): self.log.setFormatter(*args) def start(self): self.rw = os.pipe() # Open a pipe super().start() # Call supercalls start method def run(self): """ Overloads the run method; this method will run in own thread. Reads data from pipe and passes to self.log.emit Arguments: None Keyword arguments: None Returns: None """ with os.fdopen(self.rw[0]) as fid: # Open the read-end of pipe for line in iter(fid.readline, ''): # Iterate over all lines record = logging.LogRecord('', 20, '', '', line.rstrip(), '', '') self.log.emit(record) # Pass line to rotating logger if self.callback: # If call back is set self.callback(line) # Pass line to call back self.close() def close(self): """Method to clean up the pipe and logging file""" if self.rw: # If rw is set os.close(self.rw[1]) # Close the write-end of pipe self.rw = None # Set rw to None self.log.close() # Close the log def fileno(self): """Method to get underlying file pointer; in this case pipe""" if not self.rw: # If read/write pipe NOT set self.start() # Start thread return self.rw[1] # Return write-end of pipe
class MultiProcessingLogger(logging.Handler): """A multiprocessing-safe logger with built in log rolling/backups""" def __init__(self, name, mode='a', maxsize=10000, rotate=100): """Create a new MultiprocessingLogger for a file named *name* :param name: The name of the log file to be written to :param mode: The mode in which the file should be opened in :param maxsize: The number of bytes this log file should roll over at :param rotate: The number of rolled log files to keep hanging around """ logging.Handler.__init__(self) self._handler = RotatingFileHandler(name, mode, maxsize, rotate) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)
def emit(self, record): if not record.levelno == logging.DEBUG: return RotatingFileHandler.emit(self, record)
class MultiProcessingLog(logging.Handler): def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=0): logging.Handler.__init__(self) # In case our call to RotatingFileHandler blows up we first set # the _handler to None self._handler = None self._handler = RotatingFileHandler(filename, mode, maxBytes, backupCount, encoding, delay) self.queue = multiprocessing.Queue(-1) t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args # have been stringified. Removes any chance of # unpickleable things inside and possibly reduces # message size sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): if self._handler is not None: self._handler.close() logging.Handler.close(self)
class MultiProcessingLog(logging.Handler): def __init__(self, name, mode, maxsize=0, backup=1): global handler_list logging.Handler.__init__(self) self._handler = RotatingFileHandler(name, mode, maxBytes=maxsize, backupCount=backup, encoding='UTF-8') self.queue = multiprocessing.Queue(-1) if self.name not in handler_list: handler_list.append(name) try: self._handler.doRollover() except (PermissionError, ValueError): pass t = threading.Thread(target=self.receive) t.daemon = True t.start() def setFormatter(self, fmt): logging.Handler.setFormatter(self, fmt) self._handler.setFormatter(fmt) def receive(self): while True: try: record = self.queue.get() self._handler.emit(record) # print('received on pid {}'.format(os.getpid())) except (KeyboardInterrupt, SystemExit): raise except EOFError: break except: traceback.print_exc(file=sys.stderr) def send(self, s): self.queue.put_nowait(s) def _format_record(self, record): # ensure that exc_info and args have been stringified. Removes any # chance of unpickleable things inside and possibly reduces message size # sent over the pipe if record.args: record.msg = record.msg % record.args record.args = None if record.exc_info: dummy = self.format(record) record.exc_info = None return record def emit(self, record): try: s = self._format_record(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): self._handler.close() logging.Handler.close(self)