Exemple #1
0
class FlumeHandler(logging.Handler):
    def __init__(self,
                 host="localhost",
                 port=9090,
                 timeout=1000,
                 type='ng',
                 reuse=True,
                 debug=False,
                 headers=None):
        if headers is None:
            headers = {}

        # run the regular Handler __init__
        logging.Handler.__init__(self)

        # default header detection
        if 'host' not in headers:
            headers['host'] = socket.gethostname()

        self.host = host
        self.port = port
        self.timeout = timeout
        self.type = type
        self.reuse = reuse
        self.debug = debug
        self.headers = headers
        self.eventserver = FlumeEventServer(host=self.host,
                                            port=self.port,
                                            timeout=self.timeout,
                                            type=self.type,
                                            reuse=self.reuse,
                                            debug=self.debug)
        self.reconnect = self.eventserver.reconnect

    def event_ng(self, body, headers):
        return ThriftFlumeNGEvent(headers=headers, body=body)

    def event_og(self, body, fields):
        pri = PRIORITY[fields['pri']]
        dt = int(time.time() * 1000)
        ns = datetime.now().microsecond * 1000
        host = fields['host']

        del fields['pri']
        del fields['host']

        return ThriftFlumeOGEvent(timestamp=dt,
                                  priority=pri,
                                  body=body,
                                  host=host,
                                  nanos=ns,
                                  fields=fields)

    def parse_record(self, record):
        body = self.format(record)
        headers = self.headers.copy()
        try:
            msg = ast.literal_eval(body)
        except Exception:
            msg = None
        if isinstance(msg, dict):
            if 'message' in msg:
                body = msg['message']
                del msg['message']
            else:
                body = ""
            headers.update(msg)
        headers['pri'] = record.levelname.upper()
        return body, headers

    def emit(self, record):
        if isinstance(record.msg, list):
            self.emit_many(record)
        else:
            self.emit_one(record)

    def emit_one(self, record):
        try:
            event = {'ng': self.event_ng, 'og': self.event_og}
            body, headers = self.parse_record(record)
            try:
                tevent = event[self.type](body, headers)
            except KeyError:
                raise Exception('Wrong flume type specified')

            # send event
            with self.eventserver as client:
                result = self.eventserver.append(tevent, client)
                if result != 0:
                    raise Exception('Thrift error {}'.format(result))
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as e:
            record.msg = e
            self.handleError(record)

    def emit_many(self, records):
        try:
            events = []
            for obj in records.msg:
                record = logging.LogRecord(name=records.name,
                                           level=records.levelno,
                                           pathname=records.pathname,
                                           lineno=records.lineno,
                                           msg=obj,
                                           args=records.args,
                                           exc_info=records.exc_info)
                event = {'ng': self.event_ng, 'og': self.event_og}
                body, headers = self.parse_record(record)
                try:
                    tevent = event[self.type](body, headers)
                except KeyError:
                    raise Exception('Wrong flume type specified')
                else:
                    events.append(tevent)

            # send events
            with self.eventserver as client:
                result = self.eventserver.append_batch(events, client)
                if result != 0:
                    raise Exception('Thrift error {}'.format(result))
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as e:
            record = logging.LogRecord(name=records.name,
                                       level=records.levelno,
                                       pathname=records.pathname,
                                       lineno=records.lineno,
                                       msg=e,
                                       args=records.args,
                                       exc_info=records.exc_info)
            self.handleError(record)
Exemple #2
0
class FlumeHandler(logging.Handler):
    def __init__(self, host="localhost", port=9090, type='ng', headers={}):
        # run the regular Handler __init__
        logging.Handler.__init__(self)

        self.host = host
        self.port = port
        self.type = type
        self.headers = headers
        self.eventserver = FlumeEventServer(host=self.host,
                                            port=self.port,
                                            type=self.type)

    def event_ng(self):
        self.event = ThriftFlumeNGEvent(headers=self.headers, body=self.body)

    def event_og(self):
        fields = self.headers.copy()
        pri = PRIORITY[fields['pri']]
        dt = int(time.time() * 1000)
        ns = datetime.now().microsecond * 1000
        host = fields['host']

        del fields['pri']
        del fields['host']

        self.event = ThriftFlumeOGEvent(timestamp=dt,
                                        priority=pri,
                                        body=self.body,
                                        host=host,
                                        nanos=ns,
                                        fields=fields)

    def emit(self, record):
        try:
            self.body = self.format(record)
            try:
                msg = ast.literal_eval(self.body)
            except:
                msg = None

            if isinstance(msg, dict):
                if msg.has_key('message'):
                    self.body = msg['message']
                    del msg['message']
                else:
                    self.body = ""
                self.headers = msg

            if not self.headers.has_key('host'):
                self.headers['host'] = socket.gethostname()
            self.headers['pri'] = record.levelname.upper()

            event = {'ng': self.event_ng, 'og': self.event_og}
            try:
                event[self.type]()
            except KeyError:
                raise Exception('Wrong flume type specified')

            # send event
            self.eventserver.append(self.event)

        except (KeyboardInterrupt, SystemExit):
            raise
        except:
            self.handleError(record)
Exemple #3
0
class FlumeHandler(logging.Handler):
    def __init__(self, host="localhost", port=9090, timeout=1000, type='ng',
                 reuse=True, debug=False, headers={}):
        # run the regular Handler __init__
        logging.Handler.__init__(self)

        # default header detection
        if 'host' not in headers:
            headers['host'] = socket.gethostname()

        self.host = host
        self.port = port
        self.timeout = timeout
        self.type = type
        self.reuse = reuse
        self.debug = debug
        self.headers = headers
        self.eventserver = FlumeEventServer(host=self.host,
                                            port=self.port,
                                            timeout=self.timeout,
                                            type=self.type,
                                            reuse=self.reuse,
                                            debug=self.debug)
        self.reconnect = self.eventserver.reconnect

    def event_ng(self, body, headers):
        return ThriftFlumeNGEvent(headers=headers, body=body)

    def event_og(self, body, fields):
        pri = PRIORITY[fields['pri']]
        dt = int(time.time() * 1000)
        ns = datetime.now().microsecond * 1000
        host = fields['host']

        del fields['pri']
        del fields['host']

        return ThriftFlumeOGEvent(timestamp=dt,
                                  priority=pri,
                                  body=body,
                                  host=host,
                                  nanos=ns,
                                  fields=fields)

    def parse_record(self, record):
        body = self.format(record)
        headers = self.headers.copy()
        try:
            msg = ast.literal_eval(body)
        except Exception:
            msg = None
        if isinstance(msg, dict):
            if 'message' in msg:
                body = msg['message']
                del msg['message']
            else:
                body = ""
            headers.update(msg)
        headers['pri'] = record.levelname.upper()
        return body, headers

    def emit(self, record):
        if isinstance(record.msg, list):
            self.emit_many(record)
        else:
            self.emit_one(record)

    def emit_one(self, record):
        try:
            event = {'ng': self.event_ng, 'og': self.event_og}
            body, headers = self.parse_record(record)
            try:
                tevent = event[self.type](body, headers)
            except KeyError:
                raise Exception('Wrong flume type specified')

            # send event
            with self.eventserver as client:
                self.eventserver.append(tevent, client)
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as e:
            record.msg = e
            self.handleError(record)

    def emit_many(self, records):
        try:
            events = []
            for obj in records.msg:
                record = logging.LogRecord(name=records.name,
                                           level=records.levelno,
                                           pathname=records.pathname,
                                           lineno=records.lineno,
                                           msg=obj,
                                           args=records.args,
                                           exc_info=records.exc_info)
                event = {'ng': self.event_ng, 'og': self.event_og}
                body, headers = self.parse_record(record)
                try:
                    tevent = event[self.type](body, headers)
                except KeyError:
                    raise Exception('Wrong flume type specified')
                else:
                    events.append(tevent)

            # send events
            with self.eventserver as client:
                self.eventserver.append_batch(events, client)
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as e:
            record = logging.LogRecord(name=records.name,
                                       level=records.levelno,
                                       pathname=records.pathname,
                                       lineno=records.lineno,
                                       msg=e,
                                       args=records.args,
                                       exc_info=records.exc_info)
            self.handleError(record)
Exemple #4
0
class FlumeHandler(logging.Handler):
    def __init__(self, host="localhost", port=9090, type='ng', headers={}):
        # run the regular Handler __init__
        logging.Handler.__init__(self)

        self.host = host
        self.port = port
        self.type = type
        self.headers = headers
        self.eventserver = FlumeEventServer(host=self.host, port=self.port, type=self.type)

    def event_ng(self):
        self.event = ThriftFlumeNGEvent(
            headers = self.headers,
            body = self.body
        )

    def event_og(self):
        fields = self.headers.copy()
        pri = PRIORITY[fields['pri']]
        dt = int(time.time() * 1000)
        ns = datetime.now().microsecond * 1000
        host = fields['host']

        del fields['pri']
        del fields['host']

        self.event = ThriftFlumeOGEvent(
            timestamp = dt,
            priority = pri,
            body = self.body,
            host = host,
            nanos = ns,
            fields = fields
        )

    def emit(self, record):
        try:
            self.body = self.format(record)
            try:
                msg = ast.literal_eval(self.body)
            except:
                msg = None

            if isinstance(msg, dict):
                if msg.has_key('message'):
                    self.body = msg['message']
                    del msg['message']
                else:
                    self.body = ""
                self.headers = msg

            if not self.headers.has_key('host'):
                self.headers['host'] = socket.gethostname()
            self.headers['pri'] = record.levelname.upper()

            event = { 'ng': self.event_ng,
                      'og': self.event_og }
            try:
                event[self.type]()
            except KeyError:
                raise Exception('Wrong flume type specified')

            # send event
            self.eventserver.append(self.event)

        except (KeyboardInterrupt, SystemExit):
            raise
        except:
            self.handleError(record)