def test_timestamped(self):
events = [
Event(message='1', timestamp=datetime(2013, 1, 1, 23, 59, 58)),
Event(message='2', timestamp=datetime(2013, 1, 1, 23, 59, 59)),
Event(message='3', timestamp=datetime(2013, 1, 2, 0, 0, 0)),
Event(message='4', timestamp=datetime(2013, 1, 2, 0, 0, 1))
]
with TempDirectory():
self.create({
'filename': 'output-{timestamp:%Y%m%d}.log',
'compress': 'gz'
})
map(self.input.put, events)
self.waitForEmpty()
self.assertFileContents(
events[0].to_json() + '\n' + events[1].to_json() + '\n',
'output-20130101.log.1.gz')
self.assertFileContents(
events[2].to_json() + '\n' + events[3].to_json() + '\n',
'output-20130102.log')
# assert the 'fileroll' event is generated
self.assertEquals(len(events) + 1, self.output.qsize())
outputs = [self.output.get() for i in xrange(self.output.qsize())]
assertEventEquals(
self,
Event(tags=['fileroll'],
filename='output-20130101.log.1.gz',
last=events[1],
trigger=events[2]), outputs[2])
def test_good(self):
self.create({},
[Event(data=x) for x in self.good_packets])
events = self.wait(events=len(self.good_events))
for ev in self.good_events:
assertEventEquals(self, ev, events.pop(0))
def test_bad(self):
self.create({"consume": False, "on_error": "tag"}, [Event(data=x) for x in self.bad_packets])
events = self.wait(events=len(self.bad_packets))
bad_events = [Event(data=x, message="invalid syslog", tags=["error"]) for x in self.bad_packets]
for ev in bad_events:
assertEventEquals(self, ev, events.pop(0))
def test_simple(self):
function = mock.Mock()
ev = Event(data='abc123')
self.create({'function': function}, [ev])
q = self.wait()
assertEventEquals(self, Event(data='abc123'), q[0])
function.assert_called_with(ev)
def test_bad(self):
self.create({'consume': False, 'on_error': 'tag'},
[Event(data=x) for x in self.bad_packets])
events = self.wait(events=len(self.bad_packets))
bad_events = [Event(data=x, message='invalid syslog', tags=['error']) for x in self.bad_packets]
for ev in bad_events:
assertEventEquals(self, ev, events.pop(0))
def test_simple(self):
function = mock.Mock()
ev = Event(data="abc123")
self.create({"function": function}, [ev])
q = self.wait()
assertEventEquals(self, Event(data="abc123"), q[0])
function.assert_called_with(ev)
def test_allow_hosts(self):
conf = {'port': random.randint(1024, 65535),
'allow_hosts': ['127.0.0.1']}
self.create(conf)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.sendto('abc', ('', conf['port']))
q = self.waitForQueue()
assertEventEquals(self, Event(data='abc'), q[0])
def test_event(self):
conf = {'port': random.randint(1024, 65535)}
self.create(conf)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.sendto('abc', ('', conf['port']))
# yield for processing to happen
q = self.waitForQueue()
assertEventEquals(self, Event(data='abc'), q[0])
def test_event(self):
conf = {'port': random.randint(1024, 65535)}
self.create(conf)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.sendto('abc', ('', conf['port']))
# yield for processing to happen
q = self.waitForQueue()
assertEventEquals(self, Event(data='abc'), q[0])
def test_event(self):
conf = {'address': 'ipc://testipc'}
self.create(conf)
# create a zeromq socket
ctx = zmq.Context()
sock = ctx.socket(zmq.PUSH)
sock.connect(conf['address'])
sock.send('abc')
q = self.waitForQueue()
assertEventEquals(self, Event(data='abc'), q[0])
def test_event(self):
conf = {'address': 'ipc://testipc'}
self.create(conf)
# create a zeromq socket
ctx = zmq.Context()
sock = ctx.socket(zmq.PUSH)
sock.connect(conf['address'])
sock.send('abc')
q = self.waitForQueue()
assertEventEquals(self, Event(data='abc'), q[0])
def test_multiple(self):
with TempDirectory():
conf = {'path': 'test*.log'}
self.create(conf)
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
with file('test2.log', 'w') as fin:
print >> fin, 'abc'
q = self.waitForQueue(events=2)
assertEventEquals(self, Event(data='abc'), q[0])
assertEventEquals(self, Event(data='abc'), q[1])
def test_bad(self):
self.create({
'consume': False,
'on_error': 'tag'
}, [Event(data=x) for x in self.bad_packets])
events = self.wait(events=len(self.bad_packets))
bad_events = [
Event(data=x, message='invalid syslog', tags=['error'])
for x in self.bad_packets
]
for ev in bad_events:
assertEventEquals(self, ev, events.pop(0))
def test_multiple(self):
with TempDirectory():
conf = {'path': 'test*.log'}
self.create(conf)
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
with file('test2.log', 'w') as fin:
print >> fin, 'abc'
q = self.waitForQueue(events=2)
assertEventEquals(self, Event(data='abc'), q[0])
assertEventEquals(self, Event(data='abc'), q[1])
def test_nested(self):
self.create(
{
'period': 0.1,
'metrics': {
'rails.{controller}.{action}.{0}': 'timings.*'
}
}, self.events)
# 8 events expected - the above 6, and then 2 stat events
q = self.wait(events=8)
q = [i for i in q if i.stats]
q.sort(key=lambda k: k.metric)
expected = Event(
metric='rails.home.index.timings.view',
stats={
'count': 3,
'max': 2.3,
'mean': 1.5,
'median': 1.2,
'min': 1.0,
'rate': between(1, 100),
'stddev': about(2.34, 2),
'upper95': 2.19,
'upper99': 2.278
},
tags=['stat'],
)
assertEventEquals(self, expected, q[0])
expected = Event(
metric='rails.home.login.timings.view',
stats={
'count': 1,
'max': 2.0,
'mean': 2.0,
'median': 2.0,
'min': 2.0,
'rate': between(1, 100),
'stddev': 0.0,
'upper95': 2.0,
'upper99': 2.0
},
tags=['stat'],
)
assertEventEquals(self, expected, q[1])
# wait for further two 'zero' stats
q = self.wait(events=2)
def test_truncated(self):
with TempDirectory():
conf = {'path': 'test*.log'}
self.create(conf)
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
print >> fin, 'def'
gevent.sleep(0.01)
with file('test1.log', 'w') as fin:
print >> fin, 'ghi'
q = self.waitForQueue(events=3)
assertEventEquals(self, Event(data='abc'), q[0])
def test_truncated(self):
with TempDirectory():
conf = {'path': 'test*.log'}
self.create(conf)
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
print >> fin, 'def'
gevent.sleep(0.01)
with file('test1.log', 'w') as fin:
print >> fin, 'ghi'
q = self.waitForQueue(events=3)
assertEventEquals(self, Event(data='abc'), q[0])
def test_success(self):
with TempDirectory():
conf = {'path': 'test*.log'}
self.create(conf)
# create log after a short time
gevent.sleep(0.01)
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
gevent.sleep(0.01)
print >> fin, 'def'
q = self.waitForQueue(events=2)
assertEventEquals(self, Event(data='abc'), q[0])
assertEventEquals(self, Event(data='def'), q[1])
def test_max_size(self):
with TempDirectory():
self.create({'filename': 'output.log',
'max_size': 16})
map(self.input.put, self.events)
self.waitForEmpty()
self.assertFileContents(self.events[0].to_json()+'\n', 'output.log.1')
self.assertFileContents(self.events[1].to_json()+'\n', 'output.log')
# assert the 'fileroll' event is generated
self.assert_(self.output.qsize())
events = [self.output.get() for i in xrange(self.output.qsize())]
assertEventEquals(self, Event(tags=['fileroll'], filename='output.log.1', last=self.events[0], trigger=self.events[1]), events[1])
def test_success(self):
with TempDirectory():
conf = {'path': 'test*.log'}
self.create(conf)
# create log after a short time
gevent.sleep(0.01)
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
gevent.sleep(0.01)
print >> fin, 'def'
q = self.waitForQueue(events=2)
assertEventEquals(self, Event(data='abc'), q[0])
assertEventEquals(self, Event(data='def'), q[1])
def test_resume(self):
with TempDirectory():
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
conf = {'path': 'test*.log'}
self.create(conf)
q = self.waitForQueue(events=1)
assertEventEquals(self, Event(data='abc'), q[0])
with file('test1.log', 'a') as fin:
print >> fin, 'def'
self.create(conf)
q = self.waitForQueue(events=1)
assertEventEquals(self, Event(data='def'), q[0])
def test_resume(self):
with TempDirectory():
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
conf = {'path': 'test*.log'}
self.create(conf)
q = self.waitForQueue(events=1)
assertEventEquals(self, Event(data='abc'), q[0])
with file('test1.log', 'a') as fin:
print >> fin, 'def'
self.create(conf)
q = self.waitForQueue(events=1)
assertEventEquals(self, Event(data='def'), q[0])
def test_rolling(self):
with TempDirectory():
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
conf = {'path': 'test*.log'}
self.create(conf)
gevent.sleep(0.01)
os.rename('test1.log', 'test1.log.1')
with file('test1.log', 'w') as fin:
print >> fin, 'def'
q = self.waitForQueue(events=2)
assertEventEquals(self, Event(data='abc'), q[0])
assertEventEquals(self, Event(data='def'), q[1])
def test_rolling(self):
with TempDirectory():
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
conf = {'path': 'test*.log'}
self.create(conf)
gevent.sleep(0.01)
os.rename('test1.log', 'test1.log.1')
with file('test1.log', 'w') as fin:
print >> fin, 'def'
q = self.waitForQueue(events=2)
assertEventEquals(self, Event(data='abc'), q[0])
assertEventEquals(self, Event(data='def'), q[1])
def test_nested(self):
self.create({"period": 0.1, "metrics": {"rails.{controller}.{action}.{0}": "timings.*"}}, self.events)
# 8 events expected - the above 6, and then 2 stat events
q = self.wait(events=8)
q = [i for i in q if i.stats]
q.sort(key=lambda k: k.metric)
expected = Event(
metric="rails.home.index.timings.view",
stats={
"count": 3,
"max": 2.3,
"mean": 1.5,
"median": 1.2,
"min": 1.0,
"rate": between(1, 100),
"stddev": about(2.34, 2),
"upper95": 2.19,
"upper99": 2.278,
},
tags=["stat"],
)
assertEventEquals(self, expected, q[0])
expected = Event(
metric="rails.home.login.timings.view",
stats={
"count": 1,
"max": 2.0,
"mean": 2.0,
"median": 2.0,
"min": 2.0,
"rate": between(1, 100),
"stddev": 0.0,
"upper95": 2.0,
"upper99": 2.0,
},
tags=["stat"],
)
assertEventEquals(self, expected, q[1])
# wait for further two 'zero' stats
q = self.wait(events=2)
def test_nested(self):
self.create({'period': 0.1, 'metrics': {'rails.{controller}.{action}.{0}': 'timings.*'}},
self.events)
# 8 events expected - the above 6, and then 2 stat events
q = self.wait(events=8)
q = [i for i in q if i.stats]
q.sort(key=lambda k: k.metric)
expected = Event(metric='rails.home.index.timings.view',
stats={
'count': 3,
'max': 2.3,
'mean': 1.5,
'median': 1.2,
'min': 1.0,
'rate': between(1, 100),
'stddev': about(2.34, 2),
'upper95': 2.19,
'upper99': 2.278},
tags=['stat'],
)
assertEventEquals(self, expected, q[0])
expected = Event(metric='rails.home.login.timings.view',
stats={
'count': 1,
'max': 2.0,
'mean': 2.0,
'median': 2.0,
'min': 2.0,
'rate': between(1, 100),
'stddev': 0.0,
'upper95': 2.0,
'upper99': 2.0},
tags=['stat'],
)
assertEventEquals(self, expected, q[1])
# wait for further two 'zero' stats
q = self.wait(events=2)
def test_timestamped(self):
events = [
Event(message='1', timestamp=datetime(2013, 1, 1, 23, 59, 58)),
Event(message='2', timestamp=datetime(2013, 1, 1, 23, 59, 59)),
Event(message='3', timestamp=datetime(2013, 1, 2, 0, 0, 0)),
Event(message='4', timestamp=datetime(2013, 1, 2, 0, 0, 1))]
with TempDirectory():
self.create({'filename': 'output-{timestamp:%Y%m%d}.log',
'compress': 'gz'})
map(self.input.put, events)
self.waitForEmpty()
self.assertFileContents(events[0].to_json()+'\n'+events[1].to_json()+'\n', 'output-20130101.log.1.gz')
self.assertFileContents(events[2].to_json()+'\n'+events[3].to_json()+'\n', 'output-20130102.log')
# assert the 'fileroll' event is generated
self.assertEquals(len(events)+1, self.output.qsize())
outputs = [self.output.get() for i in xrange(self.output.qsize())]
assertEventEquals(self, Event(tags=['fileroll'], filename='output-20130101.log.1.gz', last=events[1], trigger=events[2]), outputs[2])
def test_max_size(self):
with TempDirectory():
self.create({'filename': 'output.log', 'max_size': 16})
map(self.input.put, self.events)
self.waitForEmpty()
self.assertFileContents(self.events[0].to_json() + '\n',
'output.log.1')
self.assertFileContents(self.events[1].to_json() + '\n',
'output.log')
# assert the 'fileroll' event is generated
self.assert_(self.output.qsize())
events = [self.output.get() for i in xrange(self.output.qsize())]
assertEventEquals(
self,
Event(tags=['fileroll'],
filename='output.log.1',
last=self.events[0],
trigger=self.events[1]), events[1])
def test_wildcard(self):
self.create({"period": 0.1, "metrics": {"rails.{controller}.{action}.{0}": "*"}}, self.events)
# 8 events expected - the above 6, and then 4 stat events
q = self.wait(events=10)
q = [i for i in q if i.stats]
q.sort(key=lambda k: k.metric)
expected = Event(
metric="rails.home.index.bytes",
stats={
"count": 3,
"rate": between(1, 100),
"max": 30159,
"min": 6926,
"median": 18150,
"mean": 18411,
"stddev": about(30789),
"upper95": 28958.1,
"upper99": 29918.82,
},
tags=["stat"],
)
assertEventEquals(self, expected, q[0])
expected = Event(
metric="rails.home.index.duration",
stats={
"count": 3,
"rate": between(1, 100),
"max": 4.0,
"min": 3.0,
"median": 3.5,
"mean": 3.5,
"stddev": 5.0,
"upper95": 3.95,
"upper99": 3.99,
},
tags=["stat"],
)
assertEventEquals(self, expected, q[1])
expected = Event(
metric="rails.home.login.duration",
stats={
"count": 1,
"rate": between(1, 100),
"max": 2.4,
"min": 2.4,
"median": 2.4,
"mean": 2.4,
"stddev": 0.0,
"upper95": 2.4,
"upper99": 2.4,
},
tags=["stat"],
)
assertEventEquals(self, expected, q[3])
def test_wildcard(self):
self.create({'period': 0.1, 'metrics': {'rails.{controller}.{action}.{0}': '*'}},
self.events)
# 8 events expected - the above 6, and then 4 stat events
q = self.wait(events=10)
q = [i for i in q if i.stats]
q.sort(key=lambda k: k.metric)
expected = Event(metric='rails.home.index.bytes',
stats={
'count': 3,
'rate': between(1, 100),
'max': 30159,
'min': 6926,
'median': 18150,
'mean': 18411,
'stddev': about(30789),
'upper95': 28958.1,
'upper99': 29918.82,
},
tags=['stat'],
)
assertEventEquals(self, expected, q[0])
expected = Event(metric='rails.home.index.duration',
stats={
'count': 3,
'rate': between(1, 100),
'max': 4.0,
'min': 3.0,
'median': 3.5,
'mean': 3.5,
'stddev': 5.0,
'upper95': 3.95,
'upper99': 3.99,
},
tags=['stat'],
)
assertEventEquals(self, expected, q[1])
expected = Event(metric='rails.home.login.duration',
stats={
'count': 1,
'rate': between(1, 100),
'max': 2.4,
'min': 2.4,
'median': 2.4,
'mean': 2.4,
'stddev': 0.0,
'upper95': 2.4,
'upper99': 2.4,
},
tags=['stat'],
)
assertEventEquals(self, expected, q[3])
def test_missing(self):
self.create({},
[Event()])
q = self.wait()
assertEventEquals(self, Event(b=False), q[0])
def test_snippet(self):
self.create({},
[Event(a=2)])
q = self.wait()
assertEventEquals(self, Event(a=2, b=2), q[0])
def test_false(self):
self.create({}, [Event(a=2)])
q = self.wait()
assertEventEquals(self, Event(a=2), q[0])
def test_missing(self):
self.create({}, [Event()])
q = self.wait()
assertEventEquals(self, Event(b=False), q[0])
def test_snippet(self):
self.create({}, [Event(a=2)])
q = self.wait()
assertEventEquals(self, Event(a=2, b=2), q[0])
def test_match(self):
self.create({"regex": r"(?P<letters>[a-z]+)(?P<numbers>\d+)"}, [Event(data="abc123")])
q = self.wait()
assertEventEquals(self, Event(letters="abc", numbers="123"), q[0])
def test_match(self):
self.create({'field': 'data'}, [Event(data='abc?a=1%2C2&b=2+3')])
q = self.wait()
assertEventEquals(self, Event(path='abc', a='1,2', b='2 3'), q[0])
def test_set(self):
self.create({"set": {"a": 2}}, [Event(a=1)])
q = self.wait()
assertEventEquals(self, Event(a=2), q[0])
def test_unset(self):
self.create({'unset': ['a', 'c']}, [Event(a=1, b=2)])
q = self.wait()
assertEventEquals(self, Event(b=2), q[0])
def test_copy(self):
self.create({'copy': {'b': 'a', 'd': 'c'}}, [Event(a=1, c=5)])
q = self.wait()
assertEventEquals(self, Event(b=1, a=1, c=5, d=5), q[0])
def test_set_format(self):
self.create({"set": {"a": "{b}.{c}"}}, [Event(b=1, c=2)])
q = self.wait()
assertEventEquals(self, Event(a="1.2", b=1, c=2), q[0])
def test_rename(self):
self.create({"rename": {"b": "a", "d": "c"}}, [Event(a=1)])
q = self.wait()
assertEventEquals(self, Event(b=1), q[0])
def test_rename(self):
self.create({'rename': {'b': 'a', 'd': 'c'}}, [Event(a=1)])
q = self.wait()
assertEventEquals(self, Event(b=1), q[0])
def test_wildcard(self):
self.create(
{
'period': 0.1,
'metrics': {
'rails.{controller}.{action}.{0}': '*'
}
}, self.events)
# 8 events expected - the above 6, and then 4 stat events
q = self.wait(events=10)
q = [i for i in q if i.stats]
q.sort(key=lambda k: k.metric)
expected = Event(
metric='rails.home.index.bytes',
stats={
'count': 3,
'rate': between(1, 100),
'max': 30159,
'min': 6926,
'median': 18150,
'mean': 18411,
'stddev': about(30789),
'upper95': 28958.1,
'upper99': 29918.82,
},
tags=['stat'],
)
assertEventEquals(self, expected, q[0])
expected = Event(
metric='rails.home.index.duration',
stats={
'count': 3,
'rate': between(1, 100),
'max': 4.0,
'min': 3.0,
'median': 3.5,
'mean': 3.5,
'stddev': 5.0,
'upper95': 3.95,
'upper99': 3.99,
},
tags=['stat'],
)
assertEventEquals(self, expected, q[1])
expected = Event(
metric='rails.home.login.duration',
stats={
'count': 1,
'rate': between(1, 100),
'max': 2.4,
'min': 2.4,
'median': 2.4,
'mean': 2.4,
'stddev': 0.0,
'upper95': 2.4,
'upper99': 2.4,
},
tags=['stat'],
)
assertEventEquals(self, expected, q[3])
def test_consume(self):
self.create({}, [Event(data='{"a": 1}')])
q = self.wait()
assertEventEquals(self, Event(a=1), q[0])
def test_good(self):
self.create({}, [Event(data=x) for x in self.good_packets])
events = self.wait(events=len(self.good_events))
for ev in self.good_events:
assertEventEquals(self, ev, events.pop(0))
def test_default(self):
self.create({}, [Event(a=3)])
q = self.wait()
assertEventEquals(self, Event(a=3, b=False), q[0])
def test_consume_false(self):
self.create({"consume": False}, [Event(data='{"a": 1}')])
q = self.wait()
assertEventEquals(self, Event(a=1, data='{"a": 1}'), q[0])
def test_true(self):
self.create({}, [Event(a=1)])
q = self.wait()
assertEventEquals(self, Event(a=1, b=1), q[0])
def test_consume(self):
self.create({}, [Event(data='{"a": 1}')])
q = self.wait()
assertEventEquals(self, Event(a=1), q[0])
def test_consume_false(self):
self.create({'consume': False}, [Event(data='{"a": 1}')])
q = self.wait()
assertEventEquals(self, Event(a=1, data='{"a": 1}'), q[0])
def test_match(self):
self.create({'regex': r'(?P<letters>[a-z]+)(?P<numbers>\d+)'},
[Event(data='abc123')])
q = self.wait()
assertEventEquals(self, Event(letters='abc', numbers='123'), q[0])
def test_false(self):
self.create({},
[Event(a=2)])
q = self.wait()
assertEventEquals(self, Event(a=2), q[0])
def test_set(self):
self.create({'set': {'a': 2}}, [Event(a=1)])
q = self.wait()
assertEventEquals(self, Event(a=2), q[0])
def test_default(self):
self.create({},
[Event(a=3)])
q = self.wait()
assertEventEquals(self, Event(a=3, b=False), q[0])
def test_set_format(self):
self.create({'set': {'a': '{b}.{c}'}}, [Event(b=1, c=2)])
q = self.wait()
assertEventEquals(self, Event(a='1.2', b=1, c=2), q[0])
def test_true(self):
self.create({},
[Event(a=1)])
q = self.wait()
assertEventEquals(self, Event(a=1, b=1), q[0])
def test_copy(self):
self.create({"copy": {"b": "a", "d": "c"}}, [Event(a=1, c=5)])
q = self.wait()
assertEventEquals(self, Event(b=1, a=1, c=5, d=5), q[0])
