class MemoryBuffer(BaseBuffer):
buffer_chunk_limit = config_param("size", 32 * 1024**2)
buffer_queue_limit = config_param("integer", 32)
flush_interval = config_param('time', 5)
def new_chunk(self, key, expire):
return MemoryBufferChunk(key, expire)
class StdoutOutput(Output):
autoflush = config_param('bool', False)
def emit(self, tag, es):
for t, record in es:
dt = datetime.fromtimestamp(t)
print("%s %s: %s" % (dt, tag, json.dumps(record)))
if self.autoflush:
sys.stdout.flush()
class ExecPipeOutput(BufferedOutput):
command = config_param('string')
def start(self):
preexec = getattr(os, 'setsid', None)
self._proc = subprocess.Popen(self.command,
shell=True,
stdin=subprocess.PIPE,
preexec_fn=preexec)
super(ExecPipeOutput, self).start()
def shutdown(self):
super(ExecPipeOutput, self).shutdown()
self._proc.stdin.close()
self._proc.wait()
self._proc = None
def write(self, chunk):
self._proc.stdin.write(chunk.read())
class ForwardInput(Input):
port = config_param('integer', 9880)
bind = config_param('string', '0.0.0.0')
def start(self):
log.info("start forward server on %s:%s", self.bind, self.port)
self._server = StreamServer((self.bind, self.port), self.on_connect)
self._server.start()
self._hbserver = HeartbeatServer((self.bind, self.port))
self._hbserver.start()
def shutdown(self):
self._hbserver.stop()
self._server.stop()
def on_message(self, msg):
tag = msg[0]
log.debug("on_message: recieved message %s", tag)
entries = msg[1]
ent_type = type(entries)
if ent_type is bytes:
Engine.emit_stream(tag, UnpackStream(entries))
elif ent_type in (list, tuple):
Engine.emit_stream(
tag,
[(e[0] or now(), e[1]) for e in entries],
)
else:
Engine.emit(tag, msg[1] or now(), msg[2])
def json_handler(self, data, sock):
decode = json.JSONDecoder().raw_decode
pos = 0
while 1:
try:
obj, pos = decode(data, pos)
self.on_message(obj)
except (ValueError, StopIteration):
data = data[pos:]
next_data = sock.recv(128 * 1024)
if not next_data:
break
data += next_data
pos = 0
def mpack_handler(self, data, sock):
unpacker = Unpacker()
unpacker.feed(data)
# default chunk size of memory buffer is 32MB
RECV_SIZE = 32 * 1024 * 1024
while 1:
for msg in unpacker:
self.on_message(msg)
next_data = sock.recv(RECV_SIZE)
if not next_data:
break
unpacker.feed(next_data)
def on_connect(self, sock, addr):
try:
data = sock.recv(128 * 1024)
if not data:
return
if data[0] in b'{[':
self.json_handler(data, sock)
else:
self.mpack_handler(data, sock)
finally:
sock.close()
class MultilogInput(Input):
dir = tag = key = config_param('string')
pos_file = config_param('string')
def start(self):
self._shutdown = False
self._proc = gevent.spawn(self.run)
def _open_log(self, inode=None, offset=None):
pat = os.path.join(self.dir, '@*.s')
current = os.path.join(self.dir, 'current')
files = glob.glob(pat) + [current]
files.sort()
target = None
if inode:
for f in files:
st = os.stat(f)
if st.st_ino == inode:
target = f
break
if not target:
target = files[0]
offset = 0
fd = _open_nonblock(target)
if offset:
os.lseek(fd, offset, 0)
return os.fstat(fd).st_ino, fd, offset
def _open_next(self, fd):
current = os.path.join(self.dir, 'current')
current_fd = _open_nonblock(current)
pat = os.path.join(self.dir, '@*.[su]')
files = glob.glob(pat)
files.sort()
files.append(current)
inode = os.fstat(fd).st_ino
os.close(fd)
for idx, f in enumerate(files):
if os.stat(f).st_ino == inode:
break
else:
log.warn("multilog: can't find before file. (inode=%d)", inode)
return os.fstat(current_fd).st_ino, current_fd
idx += 1
if len(files) == idx:
log.warn("multilog: rotate current => current?")
return os.fstat(current_fd).st_ino, current_fd
next_file = files[idx]
if next_file == current:
return os.fstat(current_fd).st_ino, current_fd
os.close(current_fd)
return os.stat(next_file).st_ino, _open_nonblock(next_file)
def is_current(self, fd):
current = os.path.join(self.dir, 'current')
current_ino = os.stat(current).st_ino
ino = os.fstat(fd).st_ino
log.debug("current inode=%d, fd inode=%d", current_ino, ino)
return ino == current_ino
def readsome(self, fd):
while 1:
try:
return os.read(fd, 1024**2)
except OSError as e:
if e.errno == errno.EAGAIN:
gevent.sleep(0.2)
continue
raise
def emit(self, line):
tai, line = line.split(' ', 1)
tai = int(tai[1:17], 16)
if tai >= 2**62:
tai -= 2**62
else:
tai = 0
Engine.emit(self.tag, tai, {self.key: line})
def run(self):
db = gdbm.open(self.pos_file, 'cs')
fd = None
before = b''
if 'pos' in db:
pos = db['pos']
inode, offset = map(int, pos.split(' '))
else:
inode = None
offset = 0
inode, fd, offset = self._open_log(inode, offset)
read_pos = offset
try:
while not self._shutdown:
buf = self.readsome(fd)
if not buf:
if self.is_current(fd):
log.debug("waiting current. offset=%d", offset)
gevent.sleep(0.3)
else:
inode, fd = self._open_next(fd)
offset = 0
continue
offset += len(buf)
if '\n' not in buf:
before += buf
continue
read_pos = offset
lines = buf.splitlines(True)
del buf
lines[0] = before + lines[0]
before = b''
last = lines[-1]
if not last.endswith('\n'):
read_pos -= len(last)
before = last
del lines[-1]
for line in lines:
self.emit(line.rstrip())
db['pos'] = "%d %d" % (inode, read_pos)
finally:
db['pos'] = '%d %d' % (inode, read_pos)
db.close()
if fd is not None:
os.close(fd)
def shutdown(self):
self._shutdown = True
self._proc.join()
class TailInput(Input):
path = tag = key = config_param('string')
rotate_wait = config_param('time', default=5)
def start(self):
self._shutdown = False
self._proc = gevent.spawn(self.run)
@property
def paths(self):
return self._paths
def configure(self, conf):
super(TailInput, self).configure(conf)
self._path = self.path.strip()
def emit(self, line):
Engine.emit(self.tag, int(time.time()), {self.key: line})
def run(self):
path = self._path
fd = None
before = b''
try:
while not self._shutdown:
if fd is None:
try:
fd = os.open(path, os.O_NONBLOCK)
log.debug("Opened log file.")
except OSError:
log.debug("Couldn't open log file.")
gevent.sleep(self.rotate_wait)
continue
buf = self.readsome(fd)
if not buf:
if self.is_current(fd):
gevent.sleep(0.3)
continue
else:
gevent.sleep(self.rotate_wait)
buf = self.readsome(fd)
if not buf:
os.close(fd)
fd = None
log.debug("Closed log file.")
continue
if b'\n' not in buf:
before += buf
continue
buf = before + buf
before = b''
lines = buf.splitlines(True)
if not lines[-1].endswith(b'\n'):
before = lines.pop()
for line in lines:
self.emit(line.rstrip())
finally:
if fd is not None:
os.close(fd)
def is_current(self, fd):
try:
ino = os.stat(self._path).st_ino
except OSError:
return False
else:
return ino == os.fstat(fd).st_ino
def readsome(self, fd):
while 1:
try:
return os.read(fd, 1024**2)
except OSError as e:
if e.errno == errno.EAGAIN:
gevent.sleep(0.2)
continue
raise
class BaseBuffer(Configurable):
buffer_chunk_limit = config_param('size', 128*1024*1024)
buffer_queue_limit = config_param('integer', 128)
flush_interval = config_param('time', 60)
_shutdown = False
def new_chunk(self, key, expire):
raise NotImplemented
def start(self):
self._queue = gevent.queue.Queue(self.buffer_queue_limit)
self._map = {}
gevent.spawn(self.run)
def run(self):
# flush を実行する間隔
check_interval = min(self.flush_interval/2, 2)
while not self._shutdown:
gevent.sleep(check_interval)
self.flush(force=False)
def emit(self, key, data):
top = self._map.get(key)
if not top:
top = self._map[key] = self.new_chunk(key, now()+self.flush_interval)
if len(top) + len(data) <= self.buffer_chunk_limit:
top += data
return False
if len(data) > self.buffer_chunk_limit:
log.warn("Size of the emitted data exceeds buffer_chunk_limit.\n"
"This may occur problems in the output plugins ``at this server.``\n"
"To avoid problems, set a smaller number to the buffer_chunk_limit\n"
"in the forward output ``at the log forwarding server.``"
)
nc = self.new_chunk(key, now()+self.flush_interval)
try:
nc += data
self._map[key] = nc
self._queue.put_nowait(top)
except gqueue.Full:
log.error("buffer_queue_limit is exceeded.")
except:
nc.purge()
raise
def keys(self):
return self._map.keys()
def flush(self, force=True):
u"""バッファリング中のチャンクを書きこみ待ちキューへ移動する.
*force* が true のときはすべてのチャンクを移動し、 false の時は
chunk.expire が古いものだけをフラッシュする.
"""
map_ = self._map
keys = list(map_.keys())
t = now()
for key in keys:
chunk = map_[key]
if not force and chunk.expire > t:
continue
del map_[key]
self._queue.put(chunk) # Would block here.
gevent.sleep(0) # give a chance to write.
if keys:
log.debug("flush: queue size=%s", self._queue.qsize())
def get(self, block=True, timeout=None):
return self._queue.get(block, timeout)
def get_nowait(self):
return self._queue.get_nowait()
def shutdown(self):
self._shutdown = True
self.flush()
class BufferedOutput(Output):
_shutdown = False
def __init__(self):
super(BufferedOutput, self).__init__()
self._last_retry_time = 0
self._next_retry_time = 0
self._secondary_limit = 8
self._emit_count = 0
buffer_type = config_param('string', 'memory')
retry_limit = config_param('integer', 17)
retry_wait = config_param('time', 1.0)
def configure(self, conf):
super(BufferedOutput, self).configure(conf)
self._buffer = Plugin.new_buffer(self.buffer_type)
self._buffer.configure(conf)
#todo: secondary
#todo: status
def start(self):
self._buffer.start()
super(BufferedOutput, self).start()
#todo: secondary.start()
gevent.spawn(self.run)
def run(self):
while not self._shutdown:
try:
chunk = self._buffer.get(timeout=1.0)
except gevent.queue.Empty:
continue
retry = 0
retry_wait = self.retry_wait
while retry <= self.retry_limit:
try:
self.write(chunk)
break
except Exception as e:
log.warn("fail to write: %r", e)
gevent.sleep(retry_wait)
retry_wait *= 2
while 1:
try:
chunk = self._buffer.get_nowait()
self.write(chunk.tag, chunk.read())
except gevent.queue.Empty:
break
def shutdown(self):
self._buffer.shutdown()
self._shutdown = True
super(BufferedOutput, self).shutdown()
def emit(self, tag, es, key=''):
self._emit_count += 1
data = self.format_stream(tag, es)
self._buffer.emit(key, data)
def write(self, chunk):
raise NotImplemented
def format(self, tag, time, record):
return "%s\t%s\t%s\n" % (time, tag, json.dumps(record))
def format_stream(self, tag, es):
buf = BytesIO()
format = self.format
for time, record in es:
buf.write(format(tag, time, record))
return buf.getvalue()