def test_json():
parser = TextParser()
parser.configure({'format': 'json'})
t, record = parser.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}')
assert t == 1362020400
assert record == {
'host' : '192.168.0.1',
'size' : 777,
'method': 'PUT',
}
def test_ltsv_cosutomized_delimiter():
parser = TextParser()
parser.configure({'format': 'ltsv', 'delimiter':',', 'label_delimiter':'='})
_, record = parser.parse('time=[28/Feb/2013:12:00:00 +0900],host=192.168.0.1,req=GET /list HTTP/1.1')
assert record == {
'time':'[28/Feb/2013:12:00:00 +0900]',
'host':'192.168.0.1',
'req' :'GET /list HTTP/1.1',
}
def test_ltsv():
parser = TextParser()
parser.configure({'format': 'ltsv'})
_, record = parser.parse("time:[28/Feb/2013:12:00:00 +0900]\thost:192.168.0.1\treq:GET /list HTTP/1.1")
assert record == {
'time':'[28/Feb/2013:12:00:00 +0900]',
'host':'192.168.0.1',
'req' :'GET /list HTTP/1.1',
}
def test_ltsv_config():
parser = TextParser()
parser.configure({'format': 'ltsv'})
assert parser.parser.delimiter == '\t'
assert parser.parser.label_delimiter == ':'
parser.configure({
'format': 'ltsv',
'delimiter': ',',
'label_delimiter': '=',
})
assert parser.parser.delimiter == ','
assert parser.parser.label_delimiter == '='
class TailInput(Input):
path = tag = config_param('string')
rotate_wait = config_param('time', default=5)
pos_file = config_param('string', None)
def __init__(self):
self._paths = []
self._pf_file = None
self._pf = None
def start(self):
self._shutdown = False
self._proc = gevent.spawn(self.run)
@property
def paths(self):
return self._paths
def configure(self, conf):
super(TailInput, self).configure(conf)
self._paths = [p.strip() for p in self.path.split(',')]
if self.pos_file:
self._pf_file = open(self.pos_file, 'r+b', 0)
self._pf = PositionFile.parse(self.pf_file)
else:
log.warn("'pos_file PATH' parameter is not set to a 'tail' source.")
log.warn("this parameter is highly recommended to save the position to resume tailing.")
self.parser = TextParser()
self.parser.configure(conf)
def receive_lines(self, lines):
lines = map(self.parser.parse, lines)
Engine.emit_stream(self.tag, lines)
def run(self):
watchers = []
threads = []
for path in self._paths:
pe = self._pf[path] if self._pf else None
w = TailWatcher(path, self.rotate_wait, pe, self.receive_lines)
watchers.append(w)
threads.append(gevent.spawn(w.run))
gevent.joinall(self.watchers)
def test_apache():
parser = TextParser()
parser.configure({'format': 'apache'})
t, record = parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0100] "GET / HTTP/1.1" 200 777')
tm = time.gmtime(t)
assert tm.tm_year == 2013
assert tm.tm_mon == 2
assert tm.tm_mday == 28
assert tm.tm_hour == 11
assert record == {
'user': '******',
'method': 'GET',
'code': '200',
'size': '777',
'host': '192.168.0.1',
'path': '/',
}
def test_nginx():
parser = TextParser()
parser.configure({'format': 'nginx'})
t, record = parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"')
tm = time.gmtime(t)
assert (tm.tm_year, tm.tm_mon, tm.tm_mday) == (2013, 2, 28)
assert (tm.tm_hour, tm.tm_min, tm.tm_sec) == (3, 0, 0)
assert record == {
'remote' : '127.0.0.1',
'host' : '192.168.0.1',
'user' : '-',
'method' : 'GET',
'path' : '/',
'code' : '200',
'size' : '777',
'referer': '-',
'agent' : 'Opera/12.0',
}
def test_syslog():
parser = TextParser()
parser.configure({'format': 'syslog'})
t, record = parser.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test')
tm = time.gmtime(t)
now = time.gmtime()
assert tm.tm_year == now.tm_year
assert tm.tm_mon == 2
assert tm.tm_mday == 28
assert tm.tm_hour == 12
assert tm.tm_min == 0
assert tm.tm_sec == 0
assert record == {
'host' : '192.168.0.1',
'ident' : 'fluentd',
'pid' : '11111',
'message': '[error] Syslog test',
}
def test_apache2():
parser = TextParser()
parser.configure({'format': 'apache2'})
t, record = parser.parse('192.168.0.1 - - [27/Feb/2013:20:00:00 -0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"')
tm = time.gmtime(t)
assert tm.tm_year == 2013
assert tm.tm_mon == 2
assert tm.tm_mday == 28
assert tm.tm_hour == 5
assert record == {
'user' : None,
'method' : 'GET',
'code' : 200,
'size' : 777,
'host' : '192.168.0.1',
'path' : '/',
'referer': None,
'agent' : 'Opera/12.0'
}
def configure(self, conf):
super(TailInput, self).configure(conf)
self._paths = [p.strip() for p in self.path.split(',')]
if self.pos_file:
self._pf_file = open(self.pos_file, 'r+b', 0)
self._pf = PositionFile.parse(self.pf_file)
else:
log.warn("'pos_file PATH' parameter is not set to a 'tail' source.")
log.warn("this parameter is highly recommended to save the position to resume tailing.")
self.parser = TextParser()
self.parser.configure(conf)
