def test_002_deactivate_normalizer(self): """ Verify that normalizer deactivation is working. """ ln = LogNormalizer(self.normalizer_path) active_n = ln.get_active_normalizers() to_deactivate = active_n.keys()[:2] for to_d in to_deactivate: del active_n[to_d] ln.set_active_normalizers(active_n) ln.reload() self.assertEqual(len([an[0] for an in ln.get_active_normalizers().items() if an[1]]), len(ln)-2) self.assertEqual(len(ln._cache), len(ln)-2)
def test_002_deactivate_normalizer(self): """ Verify that normalizer deactivation is working. """ ln = LogNormalizer(self.normalizer_path) active_n = ln.get_active_normalizers() to_deactivate = active_n.keys()[:2] for to_d in to_deactivate: del active_n[to_d] ln.set_active_normalizers(active_n) ln.reload() self.assertEqual(len([an[0] for an in ln.get_active_normalizers().items() if an[1]]), len(ln)-2) self.assertEqual(len(ln._cache), len(ln)-2)
def test_006_normalizer_test_a_syslog_log_with_syslog_deactivate(self): """ Verify that lognormalizer does not extract syslog header as tags when syslog normalizer is deactivated. """ testlog = {'raw': 'Jul 18 08:55:35 naruto app[3245]: body message'} ln = LogNormalizer(self.normalizer_path) active_n = ln.get_active_normalizers() del active_n['syslog'] ln.set_active_normalizers(active_n) ln.reload() ln.lognormalize(testlog) self.assertTrue('uuid' in testlog.keys()) self.assertFalse('date' in testlog.keys()) self.assertFalse('program' in testlog.keys())
def test_003_activate_normalizer(self): """ Verify that normalizer activation is working. """ ln = LogNormalizer(self.normalizer_path) active_n = ln.get_active_normalizers() to_deactivate = active_n.keys()[0] to_activate = to_deactivate del active_n[to_deactivate] ln.set_active_normalizers(active_n) ln.reload() # now deactivation should be done so reactivate active_n[to_activate] = True ln.set_active_normalizers(active_n) ln.reload() self.assertEqual(len([an[0] for an in ln.get_active_normalizers() if an[1]]), len(ln)) self.assertEqual(len(ln._cache), len(ln))
def test_006_normalizer_test_a_syslog_log_with_syslog_deactivate(self): """ Verify that lognormalizer does not extract syslog header as tags when syslog normalizer is deactivated. """ testlog = {'raw': 'Jul 18 08:55:35 naruto app[3245]: body message'} ln = LogNormalizer(self.normalizer_path) active_n = ln.get_active_normalizers() to_deactivate = [n for n in active_n.keys() if n.find('syslog') >= 0] for n in to_deactivate: del active_n[n] ln.set_active_normalizers(active_n) ln.reload() ln.lognormalize(testlog) self.assertTrue('uuid' in testlog.keys()) self.assertFalse('date' in testlog.keys()) self.assertFalse('program' in testlog.keys())
def test_003_activate_normalizer(self): """ Verify that normalizer activation is working. """ ln = LogNormalizer(self.normalizer_path) active_n = ln.get_active_normalizers() to_deactivate = active_n.keys()[0] to_activate = to_deactivate del active_n[to_deactivate] ln.set_active_normalizers(active_n) ln.reload() # now deactivation should be done so reactivate active_n[to_activate] = True ln.set_active_normalizers(active_n) ln.reload() self.assertEqual(len([an[0] for an in ln.get_active_normalizers() if an[1]]), len(ln)) self.assertEqual(len(ln._cache), len(ln))
import os import timeit from logsparser.lognormalizer import LogNormalizer if __name__ == "__main__": path = os.environ['NORMALIZERS_PATH'] ln = LogNormalizer(path) def test(): l = { 'raw': "<29>Jul 18 08:55:35 naruto squid[3245]: 1259844091.407 307 82.238.42.70 TCP_MISS/200 1015 GET http://www.ietf.org/css/ietf.css fbo DIRECT/64.170.98.32 text/css" } l = ln.uuidify(l) ln.normalize(l) print "Testing speed ..." t = timeit.Timer("test()", "from __main__ import test") speed = t.timeit(100000) / 100000 print "%.2f microseconds per pass, giving a theoretical speed of %i logs/s." % ( speed * 1000000, 1 / speed) print "Testing speed with minimal normalization ..." ln.set_active_normalizers({'syslog': True}) ln.reload() t = timeit.Timer("test()", "from __main__ import test") speed = t.timeit(100000) / 100000 print "%.2f microseconds per pass, giving a theoretical speed of %i logs/s." % ( speed * 1000000, 1 / speed)
# details. # # You should have received a copy of the GNU Lesser General Public License # along with this library; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import os import timeit from logsparser.lognormalizer import LogNormalizer if __name__ == "__main__": path = os.environ['NORMALIZERS_PATH'] ln = LogNormalizer(path) def test(): l = {'raw' : "<29>Jul 18 08:55:35 naruto squid[3245]: 1259844091.407 307 82.238.42.70 TCP_MISS/200 1015 GET http://www.ietf.org/css/ietf.css fbo DIRECT/64.170.98.32 text/css" } l = ln.uuidify(l) ln.normalize(l) print "Testing speed ..." t = timeit.Timer("test()", "from __main__ import test") speed = t.timeit(100000)/100000 print "%.2f microseconds per pass, giving a theoretical speed of %i logs/s." % (speed * 1000000, 1 / speed) print "Testing speed with minimal normalization ..." ln.set_active_normalizers({'syslog' : True}) ln.reload() t = timeit.Timer("test()", "from __main__ import test") speed = t.timeit(100000)/100000 print "%.2f microseconds per pass, giving a theoretical speed of %i logs/s." % (speed * 1000000, 1 / speed)