remain = sock.recv(1000000)
if not remain:
break
pos = 0
data = data[pos:] + remain
def mpack_handler(self, data, sock):
unpacker = Unpacker()
unpacker.feed(data)
while 1:
for msg in unpacker:
self.on_message(msg)
next = sock.recv(1000000)
if not next:
break
unpacker.feed(next)
def on_connect(self, sock, addr):
try:
data = sock.recv(1000000)
if not data:
return
if data[0] in b'{[':
self.json_handler(data, sock)
else:
self.mpack_handler(data, sock)
finally:
sock.close()
Plugin.register_input('forward', ForwardInput)
before += buf
continue
read_pos = offset
lines = buf.splitlines(True)
del buf
lines[0] = before + lines[0]
before = b''
last = lines[-1]
if not last.endswith('\n'):
read_pos -= len(last)
before = last
del lines[-1]
for line in lines:
self.emit(line.rstrip())
db['pos'] = "%d %d" % (inode, read_pos)
finally:
db['pos'] = '%d %d' % (inode, read_pos)
db.close()
if fd is not None:
os.close(fd)
def shutdown(self):
self._shutdown = True
self._proc.join()
Plugin.register_input('multilog', MultilogInput)
data += next_data
pos = 0
def mpack_handler(self, data, sock):
unpacker = Unpacker()
unpacker.feed(data)
# default chunk size of memory buffer is 32MB
RECV_SIZE = 32 * 1024 * 1024
while 1:
for msg in unpacker:
self.on_message(msg)
next_data = sock.recv(RECV_SIZE)
if not next_data:
break
unpacker.feed(next_data)
def on_connect(self, sock, addr):
try:
data = sock.recv(128 * 1024)
if not data:
return
if data[0] in b'{[':
self.json_handler(data, sock)
else:
self.mpack_handler(data, sock)
finally:
sock.close()
Plugin.register_input('forward', ForwardInput)
before += buf
continue
read_pos = offset
lines = buf.splitlines(True)
del buf
lines[0] = before + lines[0]
before = b''
last = lines[-1]
if not last.endswith('\n'):
read_pos -= len(last)
before = last
del lines[-1]
for line in lines:
self.emit(line.rstrip())
db['pos'] = "%d %d" % (inode, read_pos)
finally:
db['pos'] = '%d %d' % (inode, read_pos)
db.close()
if fd is not None:
os.close(fd)
def shutdown(self):
self._shutdown = True
self._proc.join()
Plugin.register_input('multilog', MultilogInput)
if not lines[-1].endswith(b'\n'):
before = lines.pop()
for line in lines:
self.emit(line.rstrip())
finally:
if fd is not None:
os.close(fd)
def is_current(self, fd):
try:
ino = os.stat(self._path).st_ino
except OSError:
return False
else:
return ino == os.fstat(fd).st_ino
def readsome(self, fd):
while 1:
try:
return os.read(fd, 1024**2)
except OSError as e:
if e.errno == errno.EAGAIN:
gevent.sleep(0.2)
continue
raise
Plugin.register_input('tail', TailInput)
elif content_type.startswith('application/json'):
params['json'] = input.read()
if 'msgpack' in params:
record = msgpack.unpackb(params['msgpack'])
elif 'json' in params:
record = json.loads(params['json'])
else:
record = params
if 'time' in params:
time_ = int(params['time'])
else:
time_ = int(time.time())
log.debug("Recieve message: tag=%r, record=%r", tag, record)
Engine.emit(tag, time_, record)
start("200 OK", [('Content-Type', 'text/plain')])
return [""]
def start(self):
log.info("start http server on %s:%s", self.bind, self.port)
self._server = server = WSGIServer((self.bind, self.port), self.wsgi_app, log=None)
server.start()
def shutdown(self):
self._server.stop()
Plugin.register_input('http', Httpinput)
