class FlumeHandler(logging.Handler): def __init__(self, host="localhost", port=9090, timeout=1000, type='ng', reuse=True, debug=False, headers=None): if headers is None: headers = {} # run the regular Handler __init__ logging.Handler.__init__(self) # default header detection if 'host' not in headers: headers['host'] = socket.gethostname() self.host = host self.port = port self.timeout = timeout self.type = type self.reuse = reuse self.debug = debug self.headers = headers self.eventserver = FlumeEventServer(host=self.host, port=self.port, timeout=self.timeout, type=self.type, reuse=self.reuse, debug=self.debug) self.reconnect = self.eventserver.reconnect def event_ng(self, body, headers): return ThriftFlumeNGEvent(headers=headers, body=body) def event_og(self, body, fields): pri = PRIORITY[fields['pri']] dt = int(time.time() * 1000) ns = datetime.now().microsecond * 1000 host = fields['host'] del fields['pri'] del fields['host'] return ThriftFlumeOGEvent(timestamp=dt, priority=pri, body=body, host=host, nanos=ns, fields=fields) def parse_record(self, record): body = self.format(record) headers = self.headers.copy() try: msg = ast.literal_eval(body) except Exception: msg = None if isinstance(msg, dict): if 'message' in msg: body = msg['message'] del msg['message'] else: body = "" headers.update(msg) headers['pri'] = record.levelname.upper() return body, headers def emit(self, record): if isinstance(record.msg, list): self.emit_many(record) else: self.emit_one(record) def emit_one(self, record): try: event = {'ng': self.event_ng, 'og': self.event_og} body, headers = self.parse_record(record) try: tevent = event[self.type](body, headers) except KeyError: raise Exception('Wrong flume type specified') # send event with self.eventserver as client: result = self.eventserver.append(tevent, client) if result != 0: raise Exception('Thrift error {}'.format(result)) except (KeyboardInterrupt, SystemExit): raise except Exception as e: record.msg = e self.handleError(record) def emit_many(self, records): try: events = [] for obj in records.msg: record = logging.LogRecord(name=records.name, level=records.levelno, pathname=records.pathname, lineno=records.lineno, msg=obj, args=records.args, exc_info=records.exc_info) event = {'ng': self.event_ng, 'og': self.event_og} body, headers = self.parse_record(record) try: tevent = event[self.type](body, headers) except KeyError: raise Exception('Wrong flume type specified') else: events.append(tevent) # send events with self.eventserver as client: result = self.eventserver.append_batch(events, client) if result != 0: raise Exception('Thrift error {}'.format(result)) except (KeyboardInterrupt, SystemExit): raise except Exception as e: record = logging.LogRecord(name=records.name, level=records.levelno, pathname=records.pathname, lineno=records.lineno, msg=e, args=records.args, exc_info=records.exc_info) self.handleError(record)
class FlumeHandler(logging.Handler): def __init__(self, host="localhost", port=9090, type='ng', headers={}): # run the regular Handler __init__ logging.Handler.__init__(self) self.host = host self.port = port self.type = type self.headers = headers self.eventserver = FlumeEventServer(host=self.host, port=self.port, type=self.type) def event_ng(self): self.event = ThriftFlumeNGEvent(headers=self.headers, body=self.body) def event_og(self): fields = self.headers.copy() pri = PRIORITY[fields['pri']] dt = int(time.time() * 1000) ns = datetime.now().microsecond * 1000 host = fields['host'] del fields['pri'] del fields['host'] self.event = ThriftFlumeOGEvent(timestamp=dt, priority=pri, body=self.body, host=host, nanos=ns, fields=fields) def emit(self, record): try: self.body = self.format(record) try: msg = ast.literal_eval(self.body) except: msg = None if isinstance(msg, dict): if msg.has_key('message'): self.body = msg['message'] del msg['message'] else: self.body = "" self.headers = msg if not self.headers.has_key('host'): self.headers['host'] = socket.gethostname() self.headers['pri'] = record.levelname.upper() event = {'ng': self.event_ng, 'og': self.event_og} try: event[self.type]() except KeyError: raise Exception('Wrong flume type specified') # send event self.eventserver.append(self.event) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record)
class FlumeHandler(logging.Handler): def __init__(self, host="localhost", port=9090, timeout=1000, type='ng', reuse=True, debug=False, headers={}): # run the regular Handler __init__ logging.Handler.__init__(self) # default header detection if 'host' not in headers: headers['host'] = socket.gethostname() self.host = host self.port = port self.timeout = timeout self.type = type self.reuse = reuse self.debug = debug self.headers = headers self.eventserver = FlumeEventServer(host=self.host, port=self.port, timeout=self.timeout, type=self.type, reuse=self.reuse, debug=self.debug) self.reconnect = self.eventserver.reconnect def event_ng(self, body, headers): return ThriftFlumeNGEvent(headers=headers, body=body) def event_og(self, body, fields): pri = PRIORITY[fields['pri']] dt = int(time.time() * 1000) ns = datetime.now().microsecond * 1000 host = fields['host'] del fields['pri'] del fields['host'] return ThriftFlumeOGEvent(timestamp=dt, priority=pri, body=body, host=host, nanos=ns, fields=fields) def parse_record(self, record): body = self.format(record) headers = self.headers.copy() try: msg = ast.literal_eval(body) except Exception: msg = None if isinstance(msg, dict): if 'message' in msg: body = msg['message'] del msg['message'] else: body = "" headers.update(msg) headers['pri'] = record.levelname.upper() return body, headers def emit(self, record): if isinstance(record.msg, list): self.emit_many(record) else: self.emit_one(record) def emit_one(self, record): try: event = {'ng': self.event_ng, 'og': self.event_og} body, headers = self.parse_record(record) try: tevent = event[self.type](body, headers) except KeyError: raise Exception('Wrong flume type specified') # send event with self.eventserver as client: self.eventserver.append(tevent, client) except (KeyboardInterrupt, SystemExit): raise except Exception as e: record.msg = e self.handleError(record) def emit_many(self, records): try: events = [] for obj in records.msg: record = logging.LogRecord(name=records.name, level=records.levelno, pathname=records.pathname, lineno=records.lineno, msg=obj, args=records.args, exc_info=records.exc_info) event = {'ng': self.event_ng, 'og': self.event_og} body, headers = self.parse_record(record) try: tevent = event[self.type](body, headers) except KeyError: raise Exception('Wrong flume type specified') else: events.append(tevent) # send events with self.eventserver as client: self.eventserver.append_batch(events, client) except (KeyboardInterrupt, SystemExit): raise except Exception as e: record = logging.LogRecord(name=records.name, level=records.levelno, pathname=records.pathname, lineno=records.lineno, msg=e, args=records.args, exc_info=records.exc_info) self.handleError(record)
class FlumeHandler(logging.Handler): def __init__(self, host="localhost", port=9090, type='ng', headers={}): # run the regular Handler __init__ logging.Handler.__init__(self) self.host = host self.port = port self.type = type self.headers = headers self.eventserver = FlumeEventServer(host=self.host, port=self.port, type=self.type) def event_ng(self): self.event = ThriftFlumeNGEvent( headers = self.headers, body = self.body ) def event_og(self): fields = self.headers.copy() pri = PRIORITY[fields['pri']] dt = int(time.time() * 1000) ns = datetime.now().microsecond * 1000 host = fields['host'] del fields['pri'] del fields['host'] self.event = ThriftFlumeOGEvent( timestamp = dt, priority = pri, body = self.body, host = host, nanos = ns, fields = fields ) def emit(self, record): try: self.body = self.format(record) try: msg = ast.literal_eval(self.body) except: msg = None if isinstance(msg, dict): if msg.has_key('message'): self.body = msg['message'] del msg['message'] else: self.body = "" self.headers = msg if not self.headers.has_key('host'): self.headers['host'] = socket.gethostname() self.headers['pri'] = record.levelname.upper() event = { 'ng': self.event_ng, 'og': self.event_og } try: event[self.type]() except KeyError: raise Exception('Wrong flume type specified') # send event self.eventserver.append(self.event) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record)