def configure(self, conf):
Plugin.load_plugins()
for elem in conf.elements:
if elem.name == 'source':
self._config_source(elem)
elif elem.name == 'match':
self._config_match(elem)
def configure(self, conf):
Plugin.load_plugins()
for elem in conf.elements:
if elem.name == 'source':
self._config_source(elem)
elif elem.name == 'match':
self._config_match(elem)
def _config_source(self, elem):
type = elem['type']
if not type:
raise ConfigError("Missing 'type' parameter on <source> directive")
log.info("adding source type=%r", type)
in_ = Plugin.new_input(type)
in_.configure(elem)
self._sources.append(in_)
def _config_source(self, elem):
type = elem['type']
if not type:
raise ConfigError("Missing 'type' parameter on <source> directive")
log.info("adding source type=%r", type)
in_ = Plugin.new_input(type)
in_.configure(elem)
self._sources.append(in_)
def _config_match(self, elem):
type = elem["type"]
pattern = elem.arg
if not type:
raise ConfigError("Missing 'type' parameter on <match %s> directive", pattern)
log.info("adding match %r => %r", pattern, type)
out = Plugin.new_output(type)
out.configure(elem)
match = Match(pattern, out)
self._matches.append(match)
def configure(self, conf):
for e in conf.elements:
if e.name != "store":
continue
type_ = e.get('type')
if type_ is None:
raise error.ConfigError("Missing 'type' parameter on <store> directive")
log.debug("adding store type=%r", type_)
output = Plugin.new_output(type_)
output.configure(e)
self._outputs.append(output)
def _config_match(self, elem):
type = elem['type']
pattern = elem.arg
if not type:
raise ConfigError(
"Missing 'type' parameter on <match %s> directive", pattern)
log.info("adding match %r => %r", pattern, type)
out = Plugin.new_output(type)
out.configure(elem)
match = Match(pattern, out)
self._matches.append(match)
def configure(self, conf):
for e in conf.elements:
if e.name != "store":
continue
type_ = e.get('type')
if type_ is None:
raise error.ConfigError(
"Missing 'type' parameter on <store> directive")
log.debug("adding store type=%r", type_)
output = Plugin.new_output(type_)
output.configure(e)
self._outputs.append(output)
~~~~~~~~~~~~~~~~~~~~~~~~~~
:copyright: (c) 2012 by INADA Naoki
:license: Apache v2
"""
from __future__ import print_function, division, absolute_import, with_statement
from fluenpy.plugin import Plugin
from fluenpy.output import Output
from fluenpy.config import config_param
from datetime import datetime
import json
import sys
class StdoutOutput(Output):
autoflush = config_param('bool', False)
def emit(self, tag, es):
for t, record in es:
dt = datetime.fromtimestamp(t)
print("%s %s: %s" % (dt, tag, json.dumps(record)))
if self.autoflush:
sys.stdout.flush()
Plugin.register_output('stdout', StdoutOutput)
class MemoryBufferChunk(BaseBufferChunk):
def __init__(self, key, expire):
super(MemoryBufferChunk, self).__init__(key, expire)
self._buf = BytesIO()
def __iadd__(self, data):
self._buf.write(data)
return self
def __len__(self):
return self._buf.tell()
def read(self):
return self._buf.getvalue()
def purge(self):
self._buf = BytesIO()
class MemoryBuffer(BaseBuffer):
buffer_chunk_limit = config_param("size", 32 * 1024**2)
buffer_queue_limit = config_param("integer", 32)
flush_interval = config_param('time', 5)
def new_chunk(self, key, expire):
return MemoryBufferChunk(key, expire)
Plugin.register_buffer('memory', MemoryBuffer)
def configure(self, conf):
super(BufferedOutput, self).configure(conf)
self._buffer = Plugin.new_buffer(self.buffer_type)
self._buffer.configure(conf)
:license: Apache v2
"""
from __future__ import print_function, division, absolute_import, with_statement
import logging
log = logging.getLogger(__name__)
from fluenpy.plugin import Plugin
from fluenpy.output import Output
from fluenpy.config import config_param
from datetime import datetime
import json
import sys
class StdoutOutput(Output):
autoflush = config_param('bool', False)
def emit(self, tag, es, chain):
for t, record in es:
dt = datetime.fromtimestamp(t)
print("%s %s: %s" % (dt, tag, json.dumps(record)))
if self.autoflush:
sys.stdout.flush()
chain.next()
Plugin.register_output('stdout', StdoutOutput)
port = int(e.get("port", DEFAULT_LISTEN_PORT))
self._nodes.append((host, port))
log.info("adding forwarding server %s:%s", host, port)
def write(self, chunk):
key = chunk.key
data = chunk.read()
log.debug("sending tag=%s data=%dbytes", key, len(data))
for node in self._nodes:
try:
self.send_data(node, key, data)
break
except Exception as e:
log.warn("fail to send data to %s: %s", node, e)
pass
else:
raise Exception("No nodes are available.")
def send_data(self, node, tag, data):
sock = socket.socket()
header = b"\x92" + msgpack.packb(tag) + b"\xdb" + struct.pack("!L", len(data))
sock.connect(node)
try:
sock.sendall(header)
sock.sendall(data)
finally:
sock.close()
Plugin.register_output("forward", ForwardOutput)
if type_ is None:
raise error.ConfigError("Missing 'type' parameter on <store> directive")
log.debug("adding store type=%r", type_)
output = Plugin.new_output(type_)
output.configure(e)
self._outputs.append(output)
def start(self):
for o in self._outputs:
o.start()
def shutdown(self):
for o in self._outputs:
try:
o.shutdown()
except Exception:
log.error("Error occured while shutdown:")
log.error(traceback.format_exc())
def emit(self, tag, es):
for o in self._outputs:
try:
o.emit(tag, es)
except Exception:
log.error("Error occured while emit:")
log.error(traceback.format_exc())
Plugin.register_output('copy', CopyOutput)
def configure(self, conf):
super(BufferedOutput, self).configure(conf)
self._buffer = Plugin.new_buffer(self.buffer_type)
self._buffer.configure(conf)
self._nodes.append((host, port))
log.info("adding forwarding server %s:%s", host, port)
def write(self, chunk):
key = chunk.key
data = chunk.read()
log.debug("sending tag=%s data=%dbytes", key, len(data))
for node in self._nodes:
try:
self.send_data(node, key, data)
break
except Exception as e:
log.warn("fail to send data to %s: %s", node, e)
pass
else:
raise Exception("No nodes are available.")
def send_data(self, node, tag, data):
sock = socket.socket()
header = b"\x92" + msgpack.packb(tag) + b"\xdb" + struct.pack(
"!L", len(data))
sock.connect(node)
try:
sock.sendall(header)
sock.sendall(data)
finally:
sock.close()
Plugin.register_output('forward', ForwardOutput)
before += buf
continue
read_pos = offset
lines = buf.splitlines(True)
del buf
lines[0] = before + lines[0]
before = b''
last = lines[-1]
if not last.endswith('\n'):
read_pos -= len(last)
before = last
del lines[-1]
for line in lines:
self.emit(line.rstrip())
db['pos'] = "%d %d" % (inode, read_pos)
finally:
db['pos'] = '%d %d' % (inode, read_pos)
db.close()
if fd is not None:
os.close(fd)
def shutdown(self):
self._shutdown = True
self._proc.join()
Plugin.register_input('multilog', MultilogInput)
before += buf
continue
read_pos = offset
lines = buf.splitlines(True)
del buf
lines[0] = before + lines[0]
before = b''
last = lines[-1]
if not last.endswith('\n'):
read_pos -= len(last)
before = last
del lines[-1]
for line in lines:
self.emit(line.rstrip())
db['pos'] = "%d %d" % (inode, read_pos)
finally:
db['pos'] = '%d %d' % (inode, read_pos)
db.close()
if fd is not None:
os.close(fd)
def shutdown(self):
self._shutdown = True
self._proc.join()
Plugin.register_input('multilog', MultilogInput)
try:
import gevent.subprocess as subprocess
except ImportError:
# may block. don't use this when gevent 1.0b3 is released.
import subprocess
class ExecPipeOutput(BufferedOutput):
command = config_param('string')
def start(self):
preexec = getattr(os, 'setsid', None)
self._proc = subprocess.Popen(self.command, shell=True,
stdin=subprocess.PIPE,
preexec_fn=preexec)
super(ExecPipeOutput, self).start()
def shutdown(self):
super(ExecPipeOutput, self).shutdown()
self._proc.stdin.close()
self._proc.wait()
self._proc = None
def write(self, chunk):
self._proc.stdin.write(chunk.read())
Plugin.register_output('exec_pipe', ExecPipeOutput)
raise error.ConfigError(
"Missing 'type' parameter on <store> directive")
log.debug("adding store type=%r", type_)
output = Plugin.new_output(type_)
output.configure(e)
self._outputs.append(output)
def start(self):
for o in self._outputs:
o.start()
def shutdown(self):
for o in self._outputs:
try:
o.shutdown()
except Exception:
log.error("Error occured while shutdown:")
log.error(traceback.format_exc())
def emit(self, tag, es):
for o in self._outputs:
try:
o.emit(tag, es)
except Exception:
log.error("Error occured while emit:")
log.error(traceback.format_exc())
Plugin.register_output('copy', CopyOutput)
data += next_data
pos = 0
def mpack_handler(self, data, sock):
unpacker = Unpacker()
unpacker.feed(data)
# default chunk size of memory buffer is 32MB
RECV_SIZE = 32 * 1024 * 1024
while 1:
for msg in unpacker:
self.on_message(msg)
next_data = sock.recv(RECV_SIZE)
if not next_data:
break
unpacker.feed(next_data)
def on_connect(self, sock, addr):
try:
data = sock.recv(128 * 1024)
if not data:
return
if data[0] in b'{[':
self.json_handler(data, sock)
else:
self.mpack_handler(data, sock)
finally:
sock.close()
Plugin.register_input('forward', ForwardInput)
# coding: utf-8
"""
fluenpy.plugins.out_null
~~~~~~~~~~~~~~~~~~~~~~~~~
:copyright: (c) 2012 by INADA Naoki
:license: Apache v2
"""
from __future__ import print_function, division, absolute_import, with_statement
import logging
log = logging.getLogger(__name__)
from fluenpy import error
from fluenpy.output import Output
from fluenpy.plugin import Plugin
class NullOutput(Output):
def emit(self, tag, es):
pass
Plugin.register_output('null', NullOutput)
try:
import gevent.subprocess as subprocess
except ImportError:
# may block. don't use this when gevent 1.0b3 is released.
import subprocess
class ExecPipeOutput(BufferedOutput):
command = config_param('string')
def start(self):
preexec = getattr(os, 'setsid', None)
self._proc = subprocess.Popen(self.command,
shell=True,
stdin=subprocess.PIPE,
preexec_fn=preexec)
super(ExecPipeOutput, self).start()
def shutdown(self):
super(ExecPipeOutput, self).shutdown()
self._proc.stdin.close()
self._proc.wait()
self._proc = None
def write(self, chunk):
self._proc.stdin.write(chunk.read())
Plugin.register_output('exec_pipe', ExecPipeOutput)
if not lines[-1].endswith(b'\n'):
before = lines.pop()
for line in lines:
self.emit(line.rstrip())
finally:
if fd is not None:
os.close(fd)
def is_current(self, fd):
try:
ino = os.stat(self._path).st_ino
except OSError:
return False
else:
return ino == os.fstat(fd).st_ino
def readsome(self, fd):
while 1:
try:
return os.read(fd, 1024**2)
except OSError as e:
if e.errno == errno.EAGAIN:
gevent.sleep(0.2)
continue
raise
Plugin.register_input('tail', TailInput)
remain = sock.recv(1000000)
if not remain:
break
pos = 0
data = data[pos:] + remain
def mpack_handler(self, data, sock):
unpacker = Unpacker()
unpacker.feed(data)
while 1:
for msg in unpacker:
self.on_message(msg)
next = sock.recv(1000000)
if not next:
break
unpacker.feed(next)
def on_connect(self, sock, addr):
try:
data = sock.recv(1000000)
if not data:
return
if data[0] in b'{[':
self.json_handler(data, sock)
else:
self.mpack_handler(data, sock)
finally:
sock.close()
Plugin.register_input('forward', ForwardInput)
elif content_type.startswith('application/json'):
params['json'] = input.read()
if 'msgpack' in params:
record = msgpack.unpackb(params['msgpack'])
elif 'json' in params:
record = json.loads(params['json'])
else:
record = params
if 'time' in params:
time_ = int(params['time'])
else:
time_ = int(time.time())
log.debug("Recieve message: tag=%r, record=%r", tag, record)
Engine.emit(tag, time_, record)
start("200 OK", [('Content-Type', 'text/plain')])
return [""]
def start(self):
log.info("start http server on %s:%s", self.bind, self.port)
self._server = server = WSGIServer((self.bind, self.port), self.wsgi_app, log=None)
server.start()
def shutdown(self):
self._server.stop()
Plugin.register_input('http', Httpinput)
