def test_timestamped(self): events = [ Event(message='1', timestamp=datetime(2013, 1, 1, 23, 59, 58)), Event(message='2', timestamp=datetime(2013, 1, 1, 23, 59, 59)), Event(message='3', timestamp=datetime(2013, 1, 2, 0, 0, 0)), Event(message='4', timestamp=datetime(2013, 1, 2, 0, 0, 1)) ] with TempDirectory(): self.create({ 'filename': 'output-{timestamp:%Y%m%d}.log', 'compress': 'gz' }) map(self.input.put, events) self.waitForEmpty() self.assertFileContents( events[0].to_json() + '\n' + events[1].to_json() + '\n', 'output-20130101.log.1.gz') self.assertFileContents( events[2].to_json() + '\n' + events[3].to_json() + '\n', 'output-20130102.log') # assert the 'fileroll' event is generated self.assertEquals(len(events) + 1, self.output.qsize()) outputs = [self.output.get() for i in xrange(self.output.qsize())] assertEventEquals( self, Event(tags=['fileroll'], filename='output-20130101.log.1.gz', last=events[1], trigger=events[2]), outputs[2])
def test_good(self): self.create({}, [Event(data=x) for x in self.good_packets]) events = self.wait(events=len(self.good_events)) for ev in self.good_events: assertEventEquals(self, ev, events.pop(0))
def test_bad(self): self.create({"consume": False, "on_error": "tag"}, [Event(data=x) for x in self.bad_packets]) events = self.wait(events=len(self.bad_packets)) bad_events = [Event(data=x, message="invalid syslog", tags=["error"]) for x in self.bad_packets] for ev in bad_events: assertEventEquals(self, ev, events.pop(0))
def test_simple(self): function = mock.Mock() ev = Event(data='abc123') self.create({'function': function}, [ev]) q = self.wait() assertEventEquals(self, Event(data='abc123'), q[0]) function.assert_called_with(ev)
def test_bad(self): self.create({'consume': False, 'on_error': 'tag'}, [Event(data=x) for x in self.bad_packets]) events = self.wait(events=len(self.bad_packets)) bad_events = [Event(data=x, message='invalid syslog', tags=['error']) for x in self.bad_packets] for ev in bad_events: assertEventEquals(self, ev, events.pop(0))
def test_simple(self): function = mock.Mock() ev = Event(data="abc123") self.create({"function": function}, [ev]) q = self.wait() assertEventEquals(self, Event(data="abc123"), q[0]) function.assert_called_with(ev)
def test_allow_hosts(self): conf = {'port': random.randint(1024, 65535), 'allow_hosts': ['127.0.0.1']} self.create(conf) self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sock.sendto('abc', ('', conf['port'])) q = self.waitForQueue() assertEventEquals(self, Event(data='abc'), q[0])
def test_event(self): conf = {'port': random.randint(1024, 65535)} self.create(conf) self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sock.sendto('abc', ('', conf['port'])) # yield for processing to happen q = self.waitForQueue() assertEventEquals(self, Event(data='abc'), q[0])
def test_event(self): conf = {'address': 'ipc://testipc'} self.create(conf) # create a zeromq socket ctx = zmq.Context() sock = ctx.socket(zmq.PUSH) sock.connect(conf['address']) sock.send('abc') q = self.waitForQueue() assertEventEquals(self, Event(data='abc'), q[0])
def test_multiple(self): with TempDirectory(): conf = {'path': 'test*.log'} self.create(conf) with file('test1.log', 'w') as fin: print >> fin, 'abc' with file('test2.log', 'w') as fin: print >> fin, 'abc' q = self.waitForQueue(events=2) assertEventEquals(self, Event(data='abc'), q[0]) assertEventEquals(self, Event(data='abc'), q[1])
def test_bad(self): self.create({ 'consume': False, 'on_error': 'tag' }, [Event(data=x) for x in self.bad_packets]) events = self.wait(events=len(self.bad_packets)) bad_events = [ Event(data=x, message='invalid syslog', tags=['error']) for x in self.bad_packets ] for ev in bad_events: assertEventEquals(self, ev, events.pop(0))
def test_nested(self): self.create( { 'period': 0.1, 'metrics': { 'rails.{controller}.{action}.{0}': 'timings.*' } }, self.events) # 8 events expected - the above 6, and then 2 stat events q = self.wait(events=8) q = [i for i in q if i.stats] q.sort(key=lambda k: k.metric) expected = Event( metric='rails.home.index.timings.view', stats={ 'count': 3, 'max': 2.3, 'mean': 1.5, 'median': 1.2, 'min': 1.0, 'rate': between(1, 100), 'stddev': about(2.34, 2), 'upper95': 2.19, 'upper99': 2.278 }, tags=['stat'], ) assertEventEquals(self, expected, q[0]) expected = Event( metric='rails.home.login.timings.view', stats={ 'count': 1, 'max': 2.0, 'mean': 2.0, 'median': 2.0, 'min': 2.0, 'rate': between(1, 100), 'stddev': 0.0, 'upper95': 2.0, 'upper99': 2.0 }, tags=['stat'], ) assertEventEquals(self, expected, q[1]) # wait for further two 'zero' stats q = self.wait(events=2)
def test_truncated(self): with TempDirectory(): conf = {'path': 'test*.log'} self.create(conf) with file('test1.log', 'w') as fin: print >> fin, 'abc' print >> fin, 'def' gevent.sleep(0.01) with file('test1.log', 'w') as fin: print >> fin, 'ghi' q = self.waitForQueue(events=3) assertEventEquals(self, Event(data='abc'), q[0])
def test_success(self): with TempDirectory(): conf = {'path': 'test*.log'} self.create(conf) # create log after a short time gevent.sleep(0.01) with file('test1.log', 'w') as fin: print >> fin, 'abc' gevent.sleep(0.01) print >> fin, 'def' q = self.waitForQueue(events=2) assertEventEquals(self, Event(data='abc'), q[0]) assertEventEquals(self, Event(data='def'), q[1])
def test_max_size(self): with TempDirectory(): self.create({'filename': 'output.log', 'max_size': 16}) map(self.input.put, self.events) self.waitForEmpty() self.assertFileContents(self.events[0].to_json()+'\n', 'output.log.1') self.assertFileContents(self.events[1].to_json()+'\n', 'output.log') # assert the 'fileroll' event is generated self.assert_(self.output.qsize()) events = [self.output.get() for i in xrange(self.output.qsize())] assertEventEquals(self, Event(tags=['fileroll'], filename='output.log.1', last=self.events[0], trigger=self.events[1]), events[1])
def test_resume(self): with TempDirectory(): with file('test1.log', 'w') as fin: print >> fin, 'abc' conf = {'path': 'test*.log'} self.create(conf) q = self.waitForQueue(events=1) assertEventEquals(self, Event(data='abc'), q[0]) with file('test1.log', 'a') as fin: print >> fin, 'def' self.create(conf) q = self.waitForQueue(events=1) assertEventEquals(self, Event(data='def'), q[0])
def test_rolling(self): with TempDirectory(): with file('test1.log', 'w') as fin: print >> fin, 'abc' conf = {'path': 'test*.log'} self.create(conf) gevent.sleep(0.01) os.rename('test1.log', 'test1.log.1') with file('test1.log', 'w') as fin: print >> fin, 'def' q = self.waitForQueue(events=2) assertEventEquals(self, Event(data='abc'), q[0]) assertEventEquals(self, Event(data='def'), q[1])
def test_nested(self): self.create({"period": 0.1, "metrics": {"rails.{controller}.{action}.{0}": "timings.*"}}, self.events) # 8 events expected - the above 6, and then 2 stat events q = self.wait(events=8) q = [i for i in q if i.stats] q.sort(key=lambda k: k.metric) expected = Event( metric="rails.home.index.timings.view", stats={ "count": 3, "max": 2.3, "mean": 1.5, "median": 1.2, "min": 1.0, "rate": between(1, 100), "stddev": about(2.34, 2), "upper95": 2.19, "upper99": 2.278, }, tags=["stat"], ) assertEventEquals(self, expected, q[0]) expected = Event( metric="rails.home.login.timings.view", stats={ "count": 1, "max": 2.0, "mean": 2.0, "median": 2.0, "min": 2.0, "rate": between(1, 100), "stddev": 0.0, "upper95": 2.0, "upper99": 2.0, }, tags=["stat"], ) assertEventEquals(self, expected, q[1]) # wait for further two 'zero' stats q = self.wait(events=2)
def test_nested(self): self.create({'period': 0.1, 'metrics': {'rails.{controller}.{action}.{0}': 'timings.*'}}, self.events) # 8 events expected - the above 6, and then 2 stat events q = self.wait(events=8) q = [i for i in q if i.stats] q.sort(key=lambda k: k.metric) expected = Event(metric='rails.home.index.timings.view', stats={ 'count': 3, 'max': 2.3, 'mean': 1.5, 'median': 1.2, 'min': 1.0, 'rate': between(1, 100), 'stddev': about(2.34, 2), 'upper95': 2.19, 'upper99': 2.278}, tags=['stat'], ) assertEventEquals(self, expected, q[0]) expected = Event(metric='rails.home.login.timings.view', stats={ 'count': 1, 'max': 2.0, 'mean': 2.0, 'median': 2.0, 'min': 2.0, 'rate': between(1, 100), 'stddev': 0.0, 'upper95': 2.0, 'upper99': 2.0}, tags=['stat'], ) assertEventEquals(self, expected, q[1]) # wait for further two 'zero' stats q = self.wait(events=2)
def test_timestamped(self): events = [ Event(message='1', timestamp=datetime(2013, 1, 1, 23, 59, 58)), Event(message='2', timestamp=datetime(2013, 1, 1, 23, 59, 59)), Event(message='3', timestamp=datetime(2013, 1, 2, 0, 0, 0)), Event(message='4', timestamp=datetime(2013, 1, 2, 0, 0, 1))] with TempDirectory(): self.create({'filename': 'output-{timestamp:%Y%m%d}.log', 'compress': 'gz'}) map(self.input.put, events) self.waitForEmpty() self.assertFileContents(events[0].to_json()+'\n'+events[1].to_json()+'\n', 'output-20130101.log.1.gz') self.assertFileContents(events[2].to_json()+'\n'+events[3].to_json()+'\n', 'output-20130102.log') # assert the 'fileroll' event is generated self.assertEquals(len(events)+1, self.output.qsize()) outputs = [self.output.get() for i in xrange(self.output.qsize())] assertEventEquals(self, Event(tags=['fileroll'], filename='output-20130101.log.1.gz', last=events[1], trigger=events[2]), outputs[2])
def test_max_size(self): with TempDirectory(): self.create({'filename': 'output.log', 'max_size': 16}) map(self.input.put, self.events) self.waitForEmpty() self.assertFileContents(self.events[0].to_json() + '\n', 'output.log.1') self.assertFileContents(self.events[1].to_json() + '\n', 'output.log') # assert the 'fileroll' event is generated self.assert_(self.output.qsize()) events = [self.output.get() for i in xrange(self.output.qsize())] assertEventEquals( self, Event(tags=['fileroll'], filename='output.log.1', last=self.events[0], trigger=self.events[1]), events[1])
def test_wildcard(self): self.create({"period": 0.1, "metrics": {"rails.{controller}.{action}.{0}": "*"}}, self.events) # 8 events expected - the above 6, and then 4 stat events q = self.wait(events=10) q = [i for i in q if i.stats] q.sort(key=lambda k: k.metric) expected = Event( metric="rails.home.index.bytes", stats={ "count": 3, "rate": between(1, 100), "max": 30159, "min": 6926, "median": 18150, "mean": 18411, "stddev": about(30789), "upper95": 28958.1, "upper99": 29918.82, }, tags=["stat"], ) assertEventEquals(self, expected, q[0]) expected = Event( metric="rails.home.index.duration", stats={ "count": 3, "rate": between(1, 100), "max": 4.0, "min": 3.0, "median": 3.5, "mean": 3.5, "stddev": 5.0, "upper95": 3.95, "upper99": 3.99, }, tags=["stat"], ) assertEventEquals(self, expected, q[1]) expected = Event( metric="rails.home.login.duration", stats={ "count": 1, "rate": between(1, 100), "max": 2.4, "min": 2.4, "median": 2.4, "mean": 2.4, "stddev": 0.0, "upper95": 2.4, "upper99": 2.4, }, tags=["stat"], ) assertEventEquals(self, expected, q[3])
def test_wildcard(self): self.create({'period': 0.1, 'metrics': {'rails.{controller}.{action}.{0}': '*'}}, self.events) # 8 events expected - the above 6, and then 4 stat events q = self.wait(events=10) q = [i for i in q if i.stats] q.sort(key=lambda k: k.metric) expected = Event(metric='rails.home.index.bytes', stats={ 'count': 3, 'rate': between(1, 100), 'max': 30159, 'min': 6926, 'median': 18150, 'mean': 18411, 'stddev': about(30789), 'upper95': 28958.1, 'upper99': 29918.82, }, tags=['stat'], ) assertEventEquals(self, expected, q[0]) expected = Event(metric='rails.home.index.duration', stats={ 'count': 3, 'rate': between(1, 100), 'max': 4.0, 'min': 3.0, 'median': 3.5, 'mean': 3.5, 'stddev': 5.0, 'upper95': 3.95, 'upper99': 3.99, }, tags=['stat'], ) assertEventEquals(self, expected, q[1]) expected = Event(metric='rails.home.login.duration', stats={ 'count': 1, 'rate': between(1, 100), 'max': 2.4, 'min': 2.4, 'median': 2.4, 'mean': 2.4, 'stddev': 0.0, 'upper95': 2.4, 'upper99': 2.4, }, tags=['stat'], ) assertEventEquals(self, expected, q[3])
def test_missing(self): self.create({}, [Event()]) q = self.wait() assertEventEquals(self, Event(b=False), q[0])
def test_snippet(self): self.create({}, [Event(a=2)]) q = self.wait() assertEventEquals(self, Event(a=2, b=2), q[0])
def test_false(self): self.create({}, [Event(a=2)]) q = self.wait() assertEventEquals(self, Event(a=2), q[0])
def test_match(self): self.create({"regex": r"(?P<letters>[a-z]+)(?P<numbers>\d+)"}, [Event(data="abc123")]) q = self.wait() assertEventEquals(self, Event(letters="abc", numbers="123"), q[0])
def test_match(self): self.create({'field': 'data'}, [Event(data='abc?a=1%2C2&b=2+3')]) q = self.wait() assertEventEquals(self, Event(path='abc', a='1,2', b='2 3'), q[0])
def test_set(self): self.create({"set": {"a": 2}}, [Event(a=1)]) q = self.wait() assertEventEquals(self, Event(a=2), q[0])
def test_unset(self): self.create({'unset': ['a', 'c']}, [Event(a=1, b=2)]) q = self.wait() assertEventEquals(self, Event(b=2), q[0])
def test_copy(self): self.create({'copy': {'b': 'a', 'd': 'c'}}, [Event(a=1, c=5)]) q = self.wait() assertEventEquals(self, Event(b=1, a=1, c=5, d=5), q[0])
def test_set_format(self): self.create({"set": {"a": "{b}.{c}"}}, [Event(b=1, c=2)]) q = self.wait() assertEventEquals(self, Event(a="1.2", b=1, c=2), q[0])
def test_rename(self): self.create({"rename": {"b": "a", "d": "c"}}, [Event(a=1)]) q = self.wait() assertEventEquals(self, Event(b=1), q[0])
def test_rename(self): self.create({'rename': {'b': 'a', 'd': 'c'}}, [Event(a=1)]) q = self.wait() assertEventEquals(self, Event(b=1), q[0])
def test_wildcard(self): self.create( { 'period': 0.1, 'metrics': { 'rails.{controller}.{action}.{0}': '*' } }, self.events) # 8 events expected - the above 6, and then 4 stat events q = self.wait(events=10) q = [i for i in q if i.stats] q.sort(key=lambda k: k.metric) expected = Event( metric='rails.home.index.bytes', stats={ 'count': 3, 'rate': between(1, 100), 'max': 30159, 'min': 6926, 'median': 18150, 'mean': 18411, 'stddev': about(30789), 'upper95': 28958.1, 'upper99': 29918.82, }, tags=['stat'], ) assertEventEquals(self, expected, q[0]) expected = Event( metric='rails.home.index.duration', stats={ 'count': 3, 'rate': between(1, 100), 'max': 4.0, 'min': 3.0, 'median': 3.5, 'mean': 3.5, 'stddev': 5.0, 'upper95': 3.95, 'upper99': 3.99, }, tags=['stat'], ) assertEventEquals(self, expected, q[1]) expected = Event( metric='rails.home.login.duration', stats={ 'count': 1, 'rate': between(1, 100), 'max': 2.4, 'min': 2.4, 'median': 2.4, 'mean': 2.4, 'stddev': 0.0, 'upper95': 2.4, 'upper99': 2.4, }, tags=['stat'], ) assertEventEquals(self, expected, q[3])
def test_consume(self): self.create({}, [Event(data='{"a": 1}')]) q = self.wait() assertEventEquals(self, Event(a=1), q[0])
def test_default(self): self.create({}, [Event(a=3)]) q = self.wait() assertEventEquals(self, Event(a=3, b=False), q[0])
def test_consume_false(self): self.create({"consume": False}, [Event(data='{"a": 1}')]) q = self.wait() assertEventEquals(self, Event(a=1, data='{"a": 1}'), q[0])
def test_true(self): self.create({}, [Event(a=1)]) q = self.wait() assertEventEquals(self, Event(a=1, b=1), q[0])
def test_consume_false(self): self.create({'consume': False}, [Event(data='{"a": 1}')]) q = self.wait() assertEventEquals(self, Event(a=1, data='{"a": 1}'), q[0])
def test_match(self): self.create({'regex': r'(?P<letters>[a-z]+)(?P<numbers>\d+)'}, [Event(data='abc123')]) q = self.wait() assertEventEquals(self, Event(letters='abc', numbers='123'), q[0])
def test_set(self): self.create({'set': {'a': 2}}, [Event(a=1)]) q = self.wait() assertEventEquals(self, Event(a=2), q[0])
def test_set_format(self): self.create({'set': {'a': '{b}.{c}'}}, [Event(b=1, c=2)]) q = self.wait() assertEventEquals(self, Event(a='1.2', b=1, c=2), q[0])
def test_copy(self): self.create({"copy": {"b": "a", "d": "c"}}, [Event(a=1, c=5)]) q = self.wait() assertEventEquals(self, Event(b=1, a=1, c=5, d=5), q[0])