def test_008_normalizer_multiple_paths(self):
""" Verify we can can deal with multiple normalizer paths.
"""
fdir = tempfile.mkdtemp()
sdir = tempfile.mkdtemp()
for f in os.listdir(self.normalizer_path):
path_f = os.path.join(self.normalizer_path, f)
if os.path.isfile(path_f):
shutil.copyfile(path_f, os.path.join(fdir, f))
shutil.move(os.path.join(fdir, 'postfix.xml'),
os.path.join(sdir, 'postfix.xml'))
ln = LogNormalizer([fdir, sdir])
source = ln.get_normalizer_source('postfix-0.99')
self.assertEquals(
XMLfromstring(source).getroottree().getroot().get('name'),
'postfix')
self.assertTrue(
ln.get_normalizer_path('postfix-0.99').__contains__(
os.path.basename(sdir)))
self.assertTrue(
ln.get_normalizer_path('syslog-1.0').__contains__(
os.path.basename(fdir)))
xml_src = ln.get_normalizer_source('syslog-1.0')
os.unlink(os.path.join(fdir, 'syslog.xml'))
ln.reload()
self.assertRaises(ValueError, ln.get_normalizer_path, 'syslog-1.0')
ln.update_normalizer(xml_src, dir_path=sdir)
self.assertTrue(
ln.get_normalizer_path('syslog-1.0').__contains__(
os.path.basename(sdir)))
shutil.rmtree(fdir)
shutil.rmtree(sdir)
def test_002_deactivate_normalizer(self):
""" Verify that normalizer deactivation is working.
"""
ln = LogNormalizer(self.normalizer_path)
active_n = ln.get_active_normalizers()
to_deactivate = active_n.keys()[:2]
for to_d in to_deactivate:
del active_n[to_d]
ln.set_active_normalizers(active_n)
ln.reload()
self.assertEqual(len([an[0] for an in ln.get_active_normalizers().items() if an[1]]), len(ln)-2)
self.assertEqual(len(ln._cache), len(ln)-2)
def test_002_deactivate_normalizer(self):
""" Verify that normalizer deactivation is working.
"""
ln = LogNormalizer(self.normalizer_path)
active_n = ln.get_active_normalizers()
to_deactivate = active_n.keys()[:2]
for to_d in to_deactivate:
del active_n[to_d]
ln.set_active_normalizers(active_n)
ln.reload()
self.assertEqual(len([an[0] for an in ln.get_active_normalizers().items() if an[1]]), len(ln)-2)
self.assertEqual(len(ln._cache), len(ln)-2)
def test_006_normalizer_test_a_syslog_log_with_syslog_deactivate(self):
""" Verify that lognormalizer does not extract
syslog header as tags when syslog normalizer is deactivated.
"""
testlog = {'raw': 'Jul 18 08:55:35 naruto app[3245]: body message'}
ln = LogNormalizer(self.normalizer_path)
active_n = ln.get_active_normalizers()
del active_n['syslog']
ln.set_active_normalizers(active_n)
ln.reload()
ln.lognormalize(testlog)
self.assertTrue('uuid' in testlog.keys())
self.assertFalse('date' in testlog.keys())
self.assertFalse('program' in testlog.keys())
def test_003_activate_normalizer(self):
""" Verify that normalizer activation is working.
"""
ln = LogNormalizer(self.normalizer_path)
active_n = ln.get_active_normalizers()
to_deactivate = active_n.keys()[0]
to_activate = to_deactivate
del active_n[to_deactivate]
ln.set_active_normalizers(active_n)
ln.reload()
# now deactivation should be done so reactivate
active_n[to_activate] = True
ln.set_active_normalizers(active_n)
ln.reload()
self.assertEqual(len([an[0] for an in ln.get_active_normalizers() if an[1]]), len(ln))
self.assertEqual(len(ln._cache), len(ln))
def test_006_normalizer_test_a_syslog_log_with_syslog_deactivate(self):
""" Verify that lognormalizer does not extract
syslog header as tags when syslog normalizer is deactivated.
"""
testlog = {'raw': 'Jul 18 08:55:35 naruto app[3245]: body message'}
ln = LogNormalizer(self.normalizer_path)
active_n = ln.get_active_normalizers()
to_deactivate = [n for n in active_n.keys() if n.find('syslog') >= 0]
for n in to_deactivate:
del active_n[n]
ln.set_active_normalizers(active_n)
ln.reload()
ln.lognormalize(testlog)
self.assertTrue('uuid' in testlog.keys())
self.assertFalse('date' in testlog.keys())
self.assertFalse('program' in testlog.keys())
def test_003_activate_normalizer(self):
""" Verify that normalizer activation is working.
"""
ln = LogNormalizer(self.normalizer_path)
active_n = ln.get_active_normalizers()
to_deactivate = active_n.keys()[0]
to_activate = to_deactivate
del active_n[to_deactivate]
ln.set_active_normalizers(active_n)
ln.reload()
# now deactivation should be done so reactivate
active_n[to_activate] = True
ln.set_active_normalizers(active_n)
ln.reload()
self.assertEqual(len([an[0] for an in ln.get_active_normalizers() if an[1]]), len(ln))
self.assertEqual(len(ln._cache), len(ln))
def test_008_normalizer_multiple_paths(self):
""" Verify we can can deal with multiple normalizer paths.
"""
fdir = tempfile.mkdtemp()
sdir = tempfile.mkdtemp()
for f in os.listdir(self.normalizer_path):
path_f = os.path.join(self.normalizer_path, f)
if os.path.isfile(path_f):
shutil.copyfile(path_f, os.path.join(fdir, f))
shutil.move(os.path.join(fdir, 'postfix.xml'),
os.path.join(sdir, 'postfix.xml'))
ln = LogNormalizer([fdir, sdir])
source = ln.get_normalizer_source('postfix-0.99')
self.assertEquals(XMLfromstring(source).getroottree().getroot().get('name'), 'postfix')
self.assertTrue(ln.get_normalizer_path('postfix-0.99').__contains__(os.path.basename(sdir)))
self.assertTrue(ln.get_normalizer_path('syslog-1.0').__contains__(os.path.basename(fdir)))
xml_src = ln.get_normalizer_source('syslog-1.0')
os.unlink(os.path.join(fdir, 'syslog.xml'))
ln.reload()
self.assertRaises(ValueError, ln.get_normalizer_path, 'syslog-1.0')
ln.update_normalizer(xml_src, dir_path = sdir)
self.assertTrue(ln.get_normalizer_path('syslog-1.0').__contains__(os.path.basename(sdir)))
shutil.rmtree(fdir)
shutil.rmtree(sdir)
import os
import timeit
from logsparser.lognormalizer import LogNormalizer
if __name__ == "__main__":
path = os.environ['NORMALIZERS_PATH']
ln = LogNormalizer(path)
def test():
l = {
'raw':
"<29>Jul 18 08:55:35 naruto squid[3245]: 1259844091.407 307 82.238.42.70 TCP_MISS/200 1015 GET http://www.ietf.org/css/ietf.css fbo DIRECT/64.170.98.32 text/css"
}
l = ln.uuidify(l)
ln.normalize(l)
print "Testing speed ..."
t = timeit.Timer("test()", "from __main__ import test")
speed = t.timeit(100000) / 100000
print "%.2f microseconds per pass, giving a theoretical speed of %i logs/s." % (
speed * 1000000, 1 / speed)
print "Testing speed with minimal normalization ..."
ln.set_active_normalizers({'syslog': True})
ln.reload()
t = timeit.Timer("test()", "from __main__ import test")
speed = t.timeit(100000) / 100000
print "%.2f microseconds per pass, giving a theoretical speed of %i logs/s." % (
speed * 1000000, 1 / speed)
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import os
import timeit
from logsparser.lognormalizer import LogNormalizer
if __name__ == "__main__":
path = os.environ['NORMALIZERS_PATH']
ln = LogNormalizer(path)
def test():
l = {'raw' : "<29>Jul 18 08:55:35 naruto squid[3245]: 1259844091.407 307 82.238.42.70 TCP_MISS/200 1015 GET http://www.ietf.org/css/ietf.css fbo DIRECT/64.170.98.32 text/css" }
l = ln.uuidify(l)
ln.normalize(l)
print "Testing speed ..."
t = timeit.Timer("test()", "from __main__ import test")
speed = t.timeit(100000)/100000
print "%.2f microseconds per pass, giving a theoretical speed of %i logs/s." % (speed * 1000000, 1 / speed)
print "Testing speed with minimal normalization ..."
ln.set_active_normalizers({'syslog' : True})
ln.reload()
t = timeit.Timer("test()", "from __main__ import test")
speed = t.timeit(100000)/100000
print "%.2f microseconds per pass, giving a theoretical speed of %i logs/s." % (speed * 1000000, 1 / speed)
