def test_004_normalizer_uuid(self):
     """ Verify that we get at least uuid tag
     """
     testlog = {'raw': 'a minimal log line'}
     ln = LogNormalizer(self.normalizer_path)
     ln.lognormalize(testlog)
     self.assertTrue('uuid' in testlog.keys())
Пример #2
0
 def test_007_normalizer_getsource(self):
     """ Verify we can retreive XML source
     of a normalizer.
     """
     ln = LogNormalizer(self.normalizer_path)
     source = ln.get_normalizer_source('syslog-0.99')
     self.assertEquals(XMLfromstring(source).getroottree().getroot().get('name'), 'syslog')
Пример #3
0
 def test_007_normalizer_getsource(self):
     """ Verify we can retreive XML source
     of a normalizer.
     """
     ln = LogNormalizer(self.normalizer_path)
     source = ln.get_normalizer_source('syslog-1.0')
     self.assertEquals(XMLfromstring(source).getroottree().getroot().get('name'), 'syslog')
Пример #4
0
 def test_004_normalizer_uuid(self):
     """ Verify that we get at least uuid tag
     """
     testlog = {'raw': 'a minimal log line'}
     ln = LogNormalizer(self.normalizer_path)
     ln.lognormalize(testlog)
     self.assertTrue('uuid' in testlog.keys())
Пример #5
0
 def test_001_all_normalizers_activated(self):
     """ Verify that we have all normalizer
     activated when we instanciate LogNormalizer with
     an activate dict empty.
     """
     ln = LogNormalizer(self.normalizer_path)
     self.assertTrue(len(ln))
     self.assertEqual(len([an[0] for an in ln.get_active_normalizers() if an[1]]), len(ln))
     self.assertEqual(len(ln._cache), len(ln))
Пример #6
0
 def test_001_all_normalizers_activated(self):
     """ Verify that we have all normalizer
     activated when we instanciate LogNormalizer with
     an activate dict empty.
     """
     ln = LogNormalizer(self.normalizer_path)
     self.assertTrue(len(ln))
     self.assertEqual(len([an[0] for an in ln.get_active_normalizers() if an[1]]), len(ln))
     self.assertEqual(len(ln._cache), len(ln))
 def test_005_normalizer_test_a_syslog_log(self):
     """ Verify that lognormalizer extracts
     syslog header as tags
     """
     testlog = {'raw': 'Jul 18 08:55:35 naruto app[3245]: body message'}
     ln = LogNormalizer(self.normalizer_path)
     ln.lognormalize(testlog)
     self.assertTrue('uuid' in testlog.keys())
     self.assertTrue('date' in testlog.keys())
     self.assertEqual(testlog['body'], 'body message')
     self.assertEqual(testlog['program'], 'app')
     self.assertEqual(testlog['pid'], '3245')
Пример #8
0
 def test_005_normalizer_test_a_syslog_log(self):
     """ Verify that lognormalizer extracts
     syslog header as tags
     """
     testlog = {'raw': 'Jul 18 08:55:35 naruto app[3245]: body message'}
     ln = LogNormalizer(self.normalizer_path)
     ln.lognormalize(testlog)
     self.assertTrue('uuid' in testlog.keys())
     self.assertTrue('date' in testlog.keys())
     self.assertEqual(testlog['body'], 'body message')
     self.assertEqual(testlog['program'], 'app')
     self.assertEqual(testlog['pid'], '3245')
Пример #9
0
 def test_009_normalizer_multiple_version(self):
     """ Verify we can can deal with a normalizer with more than one version.
     """
     fdir = tempfile.mkdtemp()
     shutil.copyfile(os.path.join(self.normalizer_path, 'postfix.xml'),
                     os.path.join(fdir, 'postfix.xml'))
     # Change normalizer version in fdir path
     xml = parse(os.path.join(fdir, 'postfix.xml'))
     xmln = xml.getroot()
     xmln.set('version', '1.0')
     xml.write(os.path.join(fdir, 'postfix.xml'))
     ln = LogNormalizer([self.normalizer_path, fdir])
     self.assertEquals(XMLfromstring(ln.get_normalizer_source('postfix-0.99')).getroottree().getroot().get('version'), '0.99')
     self.assertEquals(XMLfromstring(ln.get_normalizer_source('postfix-1.0')).getroottree().getroot().get('version'), '1.0')
     shutil.rmtree(fdir)
Пример #10
0
 def test_009_normalizer_multiple_version(self):
     """ Verify we can can deal with a normalizer with more than one version.
     """
     fdir = tempfile.mkdtemp()
     shutil.copyfile(os.path.join(self.normalizer_path, 'postfix.xml'),
                     os.path.join(fdir, 'postfix.xml'))
     # Change normalizer version in fdir path
     xml = parse(os.path.join(fdir, 'postfix.xml'))
     xmln = xml.getroot()
     xmln.set('version', '1.0')
     xml.write(os.path.join(fdir, 'postfix.xml'))
     ln = LogNormalizer([self.normalizer_path, fdir])
     self.assertEquals(XMLfromstring(ln.get_normalizer_source('postfix-0.99')).getroottree().getroot().get('version'), '0.99')
     self.assertEquals(XMLfromstring(ln.get_normalizer_source('postfix-1.0')).getroottree().getroot().get('version'), '1.0')
     shutil.rmtree(fdir)
Пример #11
0
 def test_002_deactivate_normalizer(self):
     """ Verify that normalizer deactivation is working.
     """
     ln = LogNormalizer(self.normalizer_path)
     active_n = ln.get_active_normalizers()
     to_deactivate = active_n.keys()[:2]
     for to_d in to_deactivate:
         del active_n[to_d]
     ln.set_active_normalizers(active_n)
     ln.reload()
     self.assertEqual(len([an[0] for an in ln.get_active_normalizers().items() if an[1]]), len(ln)-2)
     self.assertEqual(len(ln._cache), len(ln)-2)
 def test_006_normalizer_test_a_syslog_log_with_syslog_deactivate(self):
     """ Verify that lognormalizer does not extract
     syslog header as tags when syslog normalizer is deactivated.
     """
     testlog = {'raw': 'Jul 18 08:55:35 naruto app[3245]: body message'}
     ln = LogNormalizer(self.normalizer_path)
     active_n = ln.get_active_normalizers()
     to_deactivate = [n for n in active_n.keys() if n.find('syslog') >= 0]
     for n in to_deactivate:
         del active_n[n]
     ln.set_active_normalizers(active_n)
     ln.reload()
     ln.lognormalize(testlog)
     self.assertTrue('uuid' in testlog.keys())
     self.assertFalse('date' in testlog.keys())
     self.assertFalse('program' in testlog.keys())
Пример #13
0
 def test_006_normalizer_test_a_syslog_log_with_syslog_deactivate(self):
     """ Verify that lognormalizer does not extract
     syslog header as tags when syslog normalizer is deactivated.
     """
     testlog = {'raw': 'Jul 18 08:55:35 naruto app[3245]: body message'}
     ln = LogNormalizer(self.normalizer_path)
     active_n = ln.get_active_normalizers()
     del active_n['syslog']
     ln.set_active_normalizers(active_n)
     ln.reload()
     ln.lognormalize(testlog)
     self.assertTrue('uuid' in testlog.keys())
     self.assertFalse('date' in testlog.keys())
     self.assertFalse('program' in testlog.keys())
Пример #14
0
 def test_002_deactivate_normalizer(self):
     """ Verify that normalizer deactivation is working.
     """
     ln = LogNormalizer(self.normalizer_path)
     active_n = ln.get_active_normalizers()
     to_deactivate = active_n.keys()[:2]
     for to_d in to_deactivate:
         del active_n[to_d]
     ln.set_active_normalizers(active_n)
     ln.reload()
     self.assertEqual(len([an[0] for an in ln.get_active_normalizers().items() if an[1]]), len(ln)-2)
     self.assertEqual(len(ln._cache), len(ln)-2)
Пример #15
0
 def test_008_normalizer_multiple_paths(self):
     """ Verify we can can deal with multiple normalizer paths.
     """
     fdir = tempfile.mkdtemp()
     sdir = tempfile.mkdtemp()
     for f in os.listdir(self.normalizer_path):
         path_f = os.path.join(self.normalizer_path, f)
         if os.path.isfile(path_f):
             shutil.copyfile(path_f, os.path.join(fdir, f))
     shutil.move(os.path.join(fdir, 'postfix.xml'), 
                 os.path.join(sdir, 'postfix.xml'))
     ln = LogNormalizer([fdir, sdir])
     source = ln.get_normalizer_source('postfix-0.99')
     self.assertEquals(XMLfromstring(source).getroottree().getroot().get('name'), 'postfix')
     self.assertTrue(ln.get_normalizer_path('postfix-0.99').__contains__(os.path.basename(sdir)))
     self.assertTrue(ln.get_normalizer_path('syslog-1.0').__contains__(os.path.basename(fdir)))
     xml_src = ln.get_normalizer_source('syslog-1.0')
     os.unlink(os.path.join(fdir, 'syslog.xml'))
     ln.reload()
     self.assertRaises(ValueError, ln.get_normalizer_path, 'syslog-1.0')
     ln.update_normalizer(xml_src, dir_path = sdir)
     self.assertTrue(ln.get_normalizer_path('syslog-1.0').__contains__(os.path.basename(sdir)))
     shutil.rmtree(fdir)
     shutil.rmtree(sdir)
 def test_008_normalizer_multiple_paths(self):
     """ Verify we can can deal with multiple normalizer paths.
     """
     fdir = tempfile.mkdtemp()
     sdir = tempfile.mkdtemp()
     for f in os.listdir(self.normalizer_path):
         path_f = os.path.join(self.normalizer_path, f)
         if os.path.isfile(path_f):
             shutil.copyfile(path_f, os.path.join(fdir, f))
     shutil.move(os.path.join(fdir, 'postfix.xml'),
                 os.path.join(sdir, 'postfix.xml'))
     ln = LogNormalizer([fdir, sdir])
     source = ln.get_normalizer_source('postfix-0.99')
     self.assertEquals(
         XMLfromstring(source).getroottree().getroot().get('name'),
         'postfix')
     self.assertTrue(
         ln.get_normalizer_path('postfix-0.99').__contains__(
             os.path.basename(sdir)))
     self.assertTrue(
         ln.get_normalizer_path('syslog-1.0').__contains__(
             os.path.basename(fdir)))
     xml_src = ln.get_normalizer_source('syslog-1.0')
     os.unlink(os.path.join(fdir, 'syslog.xml'))
     ln.reload()
     self.assertRaises(ValueError, ln.get_normalizer_path, 'syslog-1.0')
     ln.update_normalizer(xml_src, dir_path=sdir)
     self.assertTrue(
         ln.get_normalizer_path('syslog-1.0').__contains__(
             os.path.basename(sdir)))
     shutil.rmtree(fdir)
     shutil.rmtree(sdir)
Пример #17
0
 def test_003_activate_normalizer(self):
     """ Verify that normalizer activation is working.
     """
     ln = LogNormalizer(self.normalizer_path)
     active_n = ln.get_active_normalizers()
     to_deactivate = active_n.keys()[0]
     to_activate = to_deactivate
     del active_n[to_deactivate]
     ln.set_active_normalizers(active_n)
     ln.reload()
     # now deactivation should be done so reactivate
     active_n[to_activate] = True
     ln.set_active_normalizers(active_n)
     ln.reload()
     self.assertEqual(len([an[0] for an in ln.get_active_normalizers() if an[1]]), len(ln))
     self.assertEqual(len(ln._cache), len(ln))
Пример #18
0
# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

import os
import timeit
from logsparser.lognormalizer import LogNormalizer

if __name__ == "__main__":
    path = os.environ['NORMALIZERS_PATH']
    ln = LogNormalizer(path)

    def test():
        l = {'raw' : "<29>Jul 18 08:55:35 naruto squid[3245]: 1259844091.407    307 82.238.42.70 TCP_MISS/200 1015 GET http://www.ietf.org/css/ietf.css fbo DIRECT/64.170.98.32 text/css" }
        l = ln.uuidify(l)
        ln.normalize(l)
    
    print "Testing speed ..."
    t = timeit.Timer("test()", "from __main__ import test")
    speed = t.timeit(100000)/100000
    print "%.2f microseconds per pass, giving a theoretical speed of %i logs/s." % (speed * 1000000, 1 / speed) 
    
    print "Testing speed with minimal normalization ..."
    ln.set_active_normalizers({'syslog' : True})
    ln.reload()
    t = timeit.Timer("test()", "from __main__ import test")
Пример #19
0
if os.path.exists(PID_FILE_PATH):
        debugMessage(1, "Tamaauth is already running, exiting")
        exit(0)
else:
        pid_file = open(PID_FILE_PATH, "w")
        pid_file.write(str(os.getpid())+"\n")
        pid_file.close()

if len(sys.argv)>1:
    AUTH_LOG_PATH = sys.argv[1]

debugMessage(2,"Parsing "+AUTH_LOG_PATH+" as auth.log")


normalizer = LN('/usr/local/lib/python2.7/dist-packages/pylogsparser-0.4-py2.7.egg/share/logsparser/normalizers')

engine = sqlalchemy.create_engine('sqlite:///'+AUTH_DB_PATH)
Base = declarative_base()


class Event (Base):
    __tablename__ = 'events'
    
    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
    date = sqlalchemy.Column(sqlalchemy.DateTime)
    action = sqlalchemy.Column(sqlalchemy.String)
    user = sqlalchemy.Column(sqlalchemy.String)
    program = sqlalchemy.Column(sqlalchemy.String)
    source = sqlalchemy.Column(sqlalchemy.String)
    source_ip = sqlalchemy.Column(sqlalchemy.String)
# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

import os
import timeit
from logsparser.lognormalizer import LogNormalizer

if __name__ == "__main__":
    path = os.environ['NORMALIZERS_PATH']
    ln = LogNormalizer(path)

    def test():
        l = {
            'raw':
            "<29>Jul 18 08:55:35 naruto squid[3245]: 1259844091.407    307 82.238.42.70 TCP_MISS/200 1015 GET http://www.ietf.org/css/ietf.css fbo DIRECT/64.170.98.32 text/css"
        }
        l = ln.uuidify(l)
        ln.normalize(l)

    print "Testing speed ..."
    t = timeit.Timer("test()", "from __main__ import test")
    speed = t.timeit(100000) / 100000
    print "%.2f microseconds per pass, giving a theoretical speed of %i logs/s." % (
        speed * 1000000, 1 / speed)
 def test_003_activate_normalizer(self):
     """ Verify that normalizer activation is working.
     """
     ln = LogNormalizer(self.normalizer_path)
     active_n = ln.get_active_normalizers()
     to_deactivate = active_n.keys()[0]
     to_activate = to_deactivate
     del active_n[to_deactivate]
     ln.set_active_normalizers(active_n)
     ln.reload()
     # now deactivation should be done so reactivate
     active_n[to_activate] = True
     ln.set_active_normalizers(active_n)
     ln.reload()
     self.assertEqual(
         len([an[0] for an in ln.get_active_normalizers() if an[1]]),
         len(ln))
     self.assertEqual(len(ln._cache), len(ln))
 def bleh(paths):
     n = LogNormalizer(paths)
     return n
Пример #23
0
import sys
import re
import timeit
from logsparser.normalizer import Normalizer
from logsparser.lognormalizer import LogNormalizer
from lxml.etree import parse, DTD
"""Measuring normalizers validation time"""

VERSION = 0.99
iterations = 5000
excl = (  # Exclusion list, skip these files
    "common_callBacks.xml", "common_tagTypes.xml", "normalizer.dtd",
    "normalizer.template")
path = os.environ['NORMALIZERS_PATH']
norm = None  # normalizer object
ln = LogNormalizer(path)
tested_logs = None


class res:
    def __init__(self, it):
        self.nn = 0  # number of normalizers
        self.ts = 0.0  # times sum
        self.it = it  # number of iterations per normalizer
        self.rl = []  # results list (a list of dictionaries)

    def add_res(self, n, v, a, s):
        self.rl.append({
            "name": n,
            "version": v,
            "author": a,
Пример #24
0
import os
import numpy as np
import matplotlib.pyplot as plt
import pylab
# color palette
from matplotlib import cm
from logsparser.lognormalizer import LogNormalizer as LN
import GeoIP
 
normalizer = LN('/home/kura//.virtualenvs/ssh-attack-visualisation/share/logsparser/normalizers/')
auth_logs = open('/home/kura/workspace/ssh-attack-visualisation/logs/auth.log.combined', 'r')
locator = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
 
dataset = {}
for log in auth_logs:
    l = {'raw' : log[:-1] } # remove the ending \n
    normalizer.normalize(l)
    if l.get('action') == 'fail':
        key = str(l['date'].hour).rjust(2,'0') +\
              str(l['date'].minute).rjust(2,'0') +\
              str(l['date'].second).rjust(2,'0')
        dataset[key] = dataset.get(key, {})
        country_l = locator.country_code_by_addr(l['source_ip'])
        if country_l:
            country = country_l
        else:
            country = "Unknown"
        dataset[key][country] = dataset[key].get(country, 0) + 1

from mpl_toolkits.basemap import Basemap
 
Пример #25
0
if os.path.exists(PID_FILE_PATH):
    debugMessage(1, "Tamaauth is already running, exiting")
    exit(0)
else:
    pid_file = open(PID_FILE_PATH, "w")
    pid_file.write(str(os.getpid()) + "\n")
    pid_file.close()

if len(sys.argv) > 1:
    AUTH_LOG_PATH = sys.argv[1]

debugMessage(2, "Parsing " + AUTH_LOG_PATH + " as auth.log")

normalizer = LN(
    '/usr/local/lib/python2.7/dist-packages/pylogsparser-0.4-py2.7.egg/share/logsparser/normalizers'
)

engine = sqlalchemy.create_engine('sqlite:///' + AUTH_DB_PATH)
Base = declarative_base()


class Event(Base):
    __tablename__ = 'events'

    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
    date = sqlalchemy.Column(sqlalchemy.DateTime)
    action = sqlalchemy.Column(sqlalchemy.String)
    user = sqlalchemy.Column(sqlalchemy.String)
    program = sqlalchemy.Column(sqlalchemy.String)
    source = sqlalchemy.Column(sqlalchemy.String)