def __init__(self,
host="localhost",
port=9090,
timeout=1000,
type='ng',
reuse=True,
debug=False,
headers=None):
if headers is None:
headers = {}
# run the regular Handler __init__
logging.Handler.__init__(self)
# default header detection
if 'host' not in headers:
headers['host'] = socket.gethostname()
self.host = host
self.port = port
self.timeout = timeout
self.type = type
self.reuse = reuse
self.debug = debug
self.headers = headers
self.eventserver = FlumeEventServer(host=self.host,
port=self.port,
timeout=self.timeout,
type=self.type,
reuse=self.reuse,
debug=self.debug)
self.reconnect = self.eventserver.reconnect
def __init__(self, host="localhost", port=9090, type='ng', headers={}):
# run the regular Handler __init__
logging.Handler.__init__(self)
self.host = host
self.port = port
self.type = type
self.headers = headers
self.eventserver = FlumeEventServer(host=self.host,
port=self.port,
type=self.type)
def test_active_nodes_unicity(self):
""" Ensure the nodes unicity in the actives nodes list.
Fix:
2015-11-27: we use a list and we don't check the presence of the same node into the list.
"""
from flumelogger.eventserver import FlumeEventServer
from flumelogger.errors import ServerSelectionError
eventserver = FlumeEventServer(host='localhost:7777,localhost:8888')
eventserver._remove_node(node=('localhost', 7777))
self.assertEqual([('localhost', 8888)], eventserver.active_nodes)
def test_autoreconnect_failed(self):
""" Ensure the default_nodes keep safe when we remove a node.
Fix:
2015-11-27: the default and active nodes variable was link (memory ref).
"""
from flumelogger.eventserver import FlumeEventServer
from flumelogger.errors import ServerSelectionError
eventserver = FlumeEventServer(host='localhost:7777')
eventserver._remove_node(node=('localhost', 7777))
self.assertListEqual([('localhost', 7777)], eventserver.default_nodes)
def test_active_nodes_unicity(self):
""" Ensure the nodes unicity in the actives nodes list.
Fix:
2015-11-27: we use a list and we don't check the presence of the same node into the list.
"""
from flumelogger.eventserver import FlumeEventServer
from flumelogger.errors import ServerSelectionError
eventserver = FlumeEventServer(host='localhost:7777,localhost:8888')
eventserver._remove_node(node=('localhost', 7777))
self.assertEqual([('localhost', 8888)], eventserver.active_nodes)
def test_autoreconnect_failed(self):
""" Ensure the default_nodes keep safe when we remove a node.
Fix:
2015-11-27: the default and active nodes variable was link (memory ref).
"""
from flumelogger.eventserver import FlumeEventServer
from flumelogger.errors import ServerSelectionError
eventserver = FlumeEventServer(host='localhost:7777')
eventserver._remove_node(node=('localhost', 7777))
self.assertListEqual([('localhost', 7777)], eventserver.default_nodes)
def __init__(self, host="localhost", port=9090, timeout=1000, type='ng',
reuse=True, debug=False, headers=None):
if headers is None:
headers = {}
# run the regular Handler __init__
logging.Handler.__init__(self)
# default header detection
if 'host' not in headers:
headers['host'] = socket.gethostname()
self.host = host
self.port = port
self.timeout = timeout
self.type = type
self.reuse = reuse
self.debug = debug
self.headers = headers
self.eventserver = FlumeEventServer(host=self.host,
port=self.port,
timeout=self.timeout,
type=self.type,
reuse=self.reuse,
debug=self.debug)
self.reconnect = self.eventserver.reconnect
def __init__(self, host="localhost", port=9090, type='ng', headers={}):
# run the regular Handler __init__
logging.Handler.__init__(self)
self.host = host
self.port = port
self.type = type
self.headers = headers
self.eventserver = FlumeEventServer(host=self.host, port=self.port, type=self.type)
def test_connect_failed(self):
""" Ensure ServerSelectionError is raise when the application start
and no nodes is available.
Fix:
2015-11-27: the default_cycle variable block the exception.
"""
from flumelogger.eventserver import FlumeEventServer
from flumelogger.errors import ServerSelectionError
eventserver = FlumeEventServer(host='localhost:7777')
with self.assertRaises(ServerSelectionError):
with eventserver as client:
pass
def test_default(self):
from flumelogger.eventserver import FlumeEventServer
eventserver = FlumeEventServer()
self.assertEqual(eventserver.active_nodes, [('localhost', 9090)])
class FlumeHandler(logging.Handler):
def __init__(self,
host="localhost",
port=9090,
timeout=1000,
type='ng',
reuse=True,
debug=False,
headers=None):
if headers is None:
headers = {}
# run the regular Handler __init__
logging.Handler.__init__(self)
# default header detection
if 'host' not in headers:
headers['host'] = socket.gethostname()
self.host = host
self.port = port
self.timeout = timeout
self.type = type
self.reuse = reuse
self.debug = debug
self.headers = headers
self.eventserver = FlumeEventServer(host=self.host,
port=self.port,
timeout=self.timeout,
type=self.type,
reuse=self.reuse,
debug=self.debug)
self.reconnect = self.eventserver.reconnect
def event_ng(self, body, headers):
return ThriftFlumeNGEvent(headers=headers, body=body)
def event_og(self, body, fields):
pri = PRIORITY[fields['pri']]
dt = int(time.time() * 1000)
ns = datetime.now().microsecond * 1000
host = fields['host']
del fields['pri']
del fields['host']
return ThriftFlumeOGEvent(timestamp=dt,
priority=pri,
body=body,
host=host,
nanos=ns,
fields=fields)
def parse_record(self, record):
body = self.format(record)
headers = self.headers.copy()
try:
msg = ast.literal_eval(body)
except Exception:
msg = None
if isinstance(msg, dict):
if 'message' in msg:
body = msg['message']
del msg['message']
else:
body = ""
headers.update(msg)
headers['pri'] = record.levelname.upper()
return body, headers
def emit(self, record):
if isinstance(record.msg, list):
self.emit_many(record)
else:
self.emit_one(record)
def emit_one(self, record):
try:
event = {'ng': self.event_ng, 'og': self.event_og}
body, headers = self.parse_record(record)
try:
tevent = event[self.type](body, headers)
except KeyError:
raise Exception('Wrong flume type specified')
# send event
with self.eventserver as client:
result = self.eventserver.append(tevent, client)
if result != 0:
raise Exception('Thrift error {}'.format(result))
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
record.msg = e
self.handleError(record)
def emit_many(self, records):
try:
events = []
for obj in records.msg:
record = logging.LogRecord(name=records.name,
level=records.levelno,
pathname=records.pathname,
lineno=records.lineno,
msg=obj,
args=records.args,
exc_info=records.exc_info)
event = {'ng': self.event_ng, 'og': self.event_og}
body, headers = self.parse_record(record)
try:
tevent = event[self.type](body, headers)
except KeyError:
raise Exception('Wrong flume type specified')
else:
events.append(tevent)
# send events
with self.eventserver as client:
result = self.eventserver.append_batch(events, client)
if result != 0:
raise Exception('Thrift error {}'.format(result))
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
record = logging.LogRecord(name=records.name,
level=records.levelno,
pathname=records.pathname,
lineno=records.lineno,
msg=e,
args=records.args,
exc_info=records.exc_info)
self.handleError(record)
class FlumeHandler(logging.Handler):
def __init__(self, host="localhost", port=9090, timeout=1000, type='ng',
reuse=True, debug=False, headers={}):
# run the regular Handler __init__
logging.Handler.__init__(self)
# default header detection
if 'host' not in headers:
headers['host'] = socket.gethostname()
self.host = host
self.port = port
self.timeout = timeout
self.type = type
self.reuse = reuse
self.debug = debug
self.headers = headers
self.eventserver = FlumeEventServer(host=self.host,
port=self.port,
timeout=self.timeout,
type=self.type,
reuse=self.reuse,
debug=self.debug)
self.reconnect = self.eventserver.reconnect
def event_ng(self, body, headers):
return ThriftFlumeNGEvent(headers=headers, body=body)
def event_og(self, body, fields):
pri = PRIORITY[fields['pri']]
dt = int(time.time() * 1000)
ns = datetime.now().microsecond * 1000
host = fields['host']
del fields['pri']
del fields['host']
return ThriftFlumeOGEvent(timestamp=dt,
priority=pri,
body=body,
host=host,
nanos=ns,
fields=fields)
def parse_record(self, record):
body = self.format(record)
headers = self.headers.copy()
try:
msg = ast.literal_eval(body)
except Exception:
msg = None
if isinstance(msg, dict):
if 'message' in msg:
body = msg['message']
del msg['message']
else:
body = ""
headers.update(msg)
headers['pri'] = record.levelname.upper()
return body, headers
def emit(self, record):
if isinstance(record.msg, list):
self.emit_many(record)
else:
self.emit_one(record)
def emit_one(self, record):
try:
event = {'ng': self.event_ng, 'og': self.event_og}
body, headers = self.parse_record(record)
try:
tevent = event[self.type](body, headers)
except KeyError:
raise Exception('Wrong flume type specified')
# send event
with self.eventserver as client:
self.eventserver.append(tevent, client)
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
record.msg = e
self.handleError(record)
def emit_many(self, records):
try:
events = []
for obj in records.msg:
record = logging.LogRecord(name=records.name,
level=records.levelno,
pathname=records.pathname,
lineno=records.lineno,
msg=obj,
args=records.args,
exc_info=records.exc_info)
event = {'ng': self.event_ng, 'og': self.event_og}
body, headers = self.parse_record(record)
try:
tevent = event[self.type](body, headers)
except KeyError:
raise Exception('Wrong flume type specified')
else:
events.append(tevent)
# send events
with self.eventserver as client:
self.eventserver.append_batch(events, client)
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
record = logging.LogRecord(name=records.name,
level=records.levelno,
pathname=records.pathname,
lineno=records.lineno,
msg=e,
args=records.args,
exc_info=records.exc_info)
self.handleError(record)
class FlumeHandler(logging.Handler):
def __init__(self, host="localhost", port=9090, type='ng', headers={}):
# run the regular Handler __init__
logging.Handler.__init__(self)
self.host = host
self.port = port
self.type = type
self.headers = headers
self.eventserver = FlumeEventServer(host=self.host,
port=self.port,
type=self.type)
def event_ng(self):
self.event = ThriftFlumeNGEvent(headers=self.headers, body=self.body)
def event_og(self):
fields = self.headers.copy()
pri = PRIORITY[fields['pri']]
dt = int(time.time() * 1000)
ns = datetime.now().microsecond * 1000
host = fields['host']
del fields['pri']
del fields['host']
self.event = ThriftFlumeOGEvent(timestamp=dt,
priority=pri,
body=self.body,
host=host,
nanos=ns,
fields=fields)
def emit(self, record):
try:
self.body = self.format(record)
try:
msg = ast.literal_eval(self.body)
except:
msg = None
if isinstance(msg, dict):
if msg.has_key('message'):
self.body = msg['message']
del msg['message']
else:
self.body = ""
self.headers = msg
if not self.headers.has_key('host'):
self.headers['host'] = socket.gethostname()
self.headers['pri'] = record.levelname.upper()
event = {'ng': self.event_ng, 'og': self.event_og}
try:
event[self.type]()
except KeyError:
raise Exception('Wrong flume type specified')
# send event
self.eventserver.append(self.event)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def test_port(self):
from flumelogger.eventserver import FlumeEventServer
eventserver = FlumeEventServer(port=7777)
self.assertEqual(eventserver.active_nodes, [('localhost', 7777)])
def test_multiple_same_port(self):
from flumelogger.eventserver import FlumeEventServer
eventserver = FlumeEventServer(host='node1,node1:7777')
self.assertEqual(eventserver.active_nodes, [('node1', 9090),
('node1', 7777)])
def test_missing_host(self):
from flumelogger.eventserver import FlumeEventServer
from flumelogger.errors import ConfigurationError
with self.assertRaises(ConfigurationError):
eventserver = FlumeEventServer(host="")
class FlumeHandler(logging.Handler):
def __init__(self, host="localhost", port=9090, type='ng', headers={}):
# run the regular Handler __init__
logging.Handler.__init__(self)
self.host = host
self.port = port
self.type = type
self.headers = headers
self.eventserver = FlumeEventServer(host=self.host, port=self.port, type=self.type)
def event_ng(self):
self.event = ThriftFlumeNGEvent(
headers = self.headers,
body = self.body
)
def event_og(self):
fields = self.headers.copy()
pri = PRIORITY[fields['pri']]
dt = int(time.time() * 1000)
ns = datetime.now().microsecond * 1000
host = fields['host']
del fields['pri']
del fields['host']
self.event = ThriftFlumeOGEvent(
timestamp = dt,
priority = pri,
body = self.body,
host = host,
nanos = ns,
fields = fields
)
def emit(self, record):
try:
self.body = self.format(record)
try:
msg = ast.literal_eval(self.body)
except:
msg = None
if isinstance(msg, dict):
if msg.has_key('message'):
self.body = msg['message']
del msg['message']
else:
self.body = ""
self.headers = msg
if not self.headers.has_key('host'):
self.headers['host'] = socket.gethostname()
self.headers['pri'] = record.levelname.upper()
event = { 'ng': self.event_ng,
'og': self.event_og }
try:
event[self.type]()
except KeyError:
raise Exception('Wrong flume type specified')
# send event
self.eventserver.append(self.event)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def test_list_host_port(self):
from flumelogger.eventserver import FlumeEventServer
eventserver = FlumeEventServer(host=['agent1:7777'])
self.assertEqual(eventserver.active_nodes, [('agent1', 7777)])
