def test_timestamped(self):
events = [
Event(message='1', timestamp=datetime(2013, 1, 1, 23, 59, 58)),
Event(message='2', timestamp=datetime(2013, 1, 1, 23, 59, 59)),
Event(message='3', timestamp=datetime(2013, 1, 2, 0, 0, 0)),
Event(message='4', timestamp=datetime(2013, 1, 2, 0, 0, 1))
]
with TempDirectory():
self.create({
'filename': 'output-{timestamp:%Y%m%d}.log',
'compress': 'gz'
})
map(self.input.put, events)
self.waitForEmpty()
self.assertFileContents(
events[0].to_json() + '\n' + events[1].to_json() + '\n',
'output-20130101.log.1.gz')
self.assertFileContents(
events[2].to_json() + '\n' + events[3].to_json() + '\n',
'output-20130102.log')
# assert the 'fileroll' event is generated
self.assertEquals(len(events) + 1, self.output.qsize())
outputs = [self.output.get() for i in xrange(self.output.qsize())]
assertEventEquals(
self,
Event(tags=['fileroll'],
filename='output-20130101.log.1.gz',
last=events[1],
trigger=events[2]), outputs[2])
def test_simple(self):
with TempDirectory():
self.create({'filename': 'log/output_{program}.log'})
map(self.input.put, self.events)
self.waitForEmpty()
self.assertFileContents(self.events[0].to_json() + '\n',
'log/output_httpd.log')
self.assertFileContents(self.events[1].to_json() + '\n',
'log/output_ntpd.log')
def test_multiple(self):
with TempDirectory():
conf = {'path': 'test*.log'}
self.create(conf)
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
with file('test2.log', 'w') as fin:
print >> fin, 'abc'
q = self.waitForQueue(events=2)
assertEventEquals(self, Event(data='abc'), q[0])
assertEventEquals(self, Event(data='abc'), q[1])
def test_max_count(self):
with TempDirectory():
self.create({
'filename': 'output.log',
'max_size': 16,
'max_count': 2
})
map(self.input.put, self.events * 10)
self.waitForEmpty()
self.assertFileContents(self.events[1].to_json() + '\n',
'output.log')
self.assert_(not os.path.exists('output.log.3'))
def test_truncated(self):
with TempDirectory():
conf = {'path': 'test*.log'}
self.create(conf)
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
print >> fin, 'def'
gevent.sleep(0.01)
with file('test1.log', 'w') as fin:
print >> fin, 'ghi'
q = self.waitForQueue(events=3)
assertEventEquals(self, Event(data='abc'), q[0])
def test_success(self):
with TempDirectory():
conf = {'path': 'test*.log'}
self.create(conf)
# create log after a short time
gevent.sleep(0.01)
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
gevent.sleep(0.01)
print >> fin, 'def'
q = self.waitForQueue(events=2)
assertEventEquals(self, Event(data='abc'), q[0])
assertEventEquals(self, Event(data='def'), q[1])
def test_compress(self):
with TempDirectory():
self.create({
'filename': 'output.log',
'max_size': 16,
'compress': 'gz'
})
map(self.input.put, self.events)
self.waitForEmpty()
self.i.stop()
self.assertFileContents(self.events[0].to_json() + '\n',
'output.log.1.gz')
self.assertFileContents(self.events[1].to_json() + '\n',
'output.log')
def test_rolling(self):
with TempDirectory():
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
conf = {'path': 'test*.log'}
self.create(conf)
gevent.sleep(0.01)
os.rename('test1.log', 'test1.log.1')
with file('test1.log', 'w') as fin:
print >> fin, 'def'
q = self.waitForQueue(events=2)
assertEventEquals(self, Event(data='abc'), q[0])
assertEventEquals(self, Event(data='def'), q[1])
def test_resume(self):
with TempDirectory():
with file('test1.log', 'w') as fin:
print >> fin, 'abc'
conf = {'path': 'test*.log'}
self.create(conf)
q = self.waitForQueue(events=1)
assertEventEquals(self, Event(data='abc'), q[0])
with file('test1.log', 'a') as fin:
print >> fin, 'def'
self.create(conf)
q = self.waitForQueue(events=1)
assertEventEquals(self, Event(data='def'), q[0])
def test_max_size(self):
with TempDirectory():
self.create({'filename': 'output.log', 'max_size': 16})
map(self.input.put, self.events)
self.waitForEmpty()
self.assertFileContents(self.events[0].to_json() + '\n',
'output.log.1')
self.assertFileContents(self.events[1].to_json() + '\n',
'output.log')
# assert the 'fileroll' event is generated
self.assert_(self.output.qsize())
events = [self.output.get() for i in xrange(self.output.qsize())]
assertEventEquals(
self,
Event(tags=['fileroll'],
filename='output.log.1',
last=self.events[0],
trigger=self.events[1]), events[1])
def test_upload(self, mock_s3):
mock_conn = mock_s3.return_value
mock_bucket = mock_conn.get_bucket.return_value
mock_key = mock_bucket.new_key.return_value
with TempDirectory():
# create log file for it to upload
with file('output.log', 'w') as fout:
fout.write('a log file')
self.create({
'access_key': 'dummy',
'secret_key': 'dummy',
'bucket': 'bucket1',
'path': 'logs/1.json'
})
self.input.put(Event(tags=['fileroll'], filename='output.log'))
self.waitForEmpty()
mock_s3.assert_called_with('dummy', 'dummy')
mock_conn.get_bucket.assert_called_with('bucket1')
mock_bucket.new_key.assert_called_with('logs/1.json')
mock_key.set_contents_from_filename.assert_called_with('output.log')
def test_missing(self):
with TempDirectory():
conf = {'path': 'test*.log'}
self.create(conf)
self.waitForQueue(events=0)