def performance(duration=10, print_output=True):
normalizer = LogNormalizer()
normalizer.load_rules('./samples/rules.db')
runs = 0
then = time.time()
while time.time() - then < duration:
event = normalizer.normalize(APACHE_LL_STR)
event_dict = json.loads(event.as_json())
del (event)
del (event_dict)
runs += 1
if print_output:
print('Ran {} times in {} seconds for {} runs per second.'.format(
runs, duration, runs / float(duration)))
def performance(duration=10, print_output=True):
normalizer = LogNormalizer()
normalizer.load_rules('./samples/rules.db')
runs = 0
then = time.time()
while time.time() - then < duration:
event = normalizer.normalize(APACHE_LL_STR)
event_dict = json.loads(event.as_json())
del(event)
del(event_dict)
runs += 1
if print_output:
print('Ran {} times in {} seconds for {} runs per second.'.format(
runs,
duration,
runs / float(duration)))
def get_normalizer(conf=config.get_config()):
"""This returns both a normalizer as well as a list of loaded rules"""
normalization_conf = conf.liblognorm
normalizer = LogNormalizer()
loaded_rules = list()
if normalization_conf.rules_dir:
loaded_rules = load_rules(normalizer, normalization_conf.rules_dir)
return (normalizer, loaded_rules)
def test_normalizing_string(self):
normalizer = LogNormalizer()
normalizer.load_rules('./samples/rules.db')
event = normalizer.normalize(APACHE_LL_STR)
expected = {
'user_agent': 'curl/7.29.0',
'url': '-',
'bytes': '461',
'status_code': '404',
'http_version': 'HTTP/1.1',
'uri': '/test',
'method': 'GET',
'timestamp': '12/Jul/2013:19:31:24 +0000',
'b': '-',
'a': '-',
'remote_host': '127.0.0.1'
}
self.assertEqual(expected, json.loads(event.as_json()))
def test_normalizing_string(self):
normalizer = LogNormalizer()
normalizer.load_rules('./samples/rules.db')
event = normalizer.normalize(APACHE_LL_STR)
expected = {
'user_agent': 'curl/7.29.0',
'url': '-',
'bytes': '461',
'status_code': '404',
'http_version': 'HTTP/1.1',
'uri': '/test',
'method': 'GET',
'timestamp': '12/Jul/2013:19:31:24 +0000',
'b': '-',
'a': '-',
'remote_host': '127.0.0.1'
}
self.assertEqual(expected, json.loads(event.as_json()))
def test_normalizing_unicode(self):
normalizer = LogNormalizer()
normalizer.load_rules('./samples/rules.db')
event = normalizer.normalize(APACHE_LL_UNICODE)
expected = {
'user_agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64;'
' rv:18.0) Gecko/20100101 Firefox/18.0',
'url': 'http://nu.realityhub.com/wiki/index.php/Main_Page',
'bytes': '212',
'status_code': '304',
'http_version': 'HTTP/1.1',
'uri': '/wiki/skins/common/commonPrint.css?270',
'method': 'GET',
'timestamp': '13/Jan/2013:09:57:51 -0600',
'b': '-',
'a': '-',
'remote_host': '66.69.25.244'
}
self.assertEqual(expected, json.loads(event.as_json()))
def test_normalizing_unicode(self):
normalizer = LogNormalizer()
normalizer.load_rules('./samples/rules.db')
event = normalizer.normalize(APACHE_LL_UNICODE)
expected = {
'user_agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64;'
' rv:18.0) Gecko/20100101 Firefox/18.0',
'url': 'http://nu.realityhub.com/wiki/index.php/Main_Page',
'bytes': '212',
'status_code': '304',
'http_version': 'HTTP/1.1',
'uri': '/wiki/skins/common/commonPrint.css?270',
'method': 'GET',
'timestamp': '13/Jan/2013:09:57:51 -0600',
'b': '-',
'a': '-',
'remote_host': '66.69.25.244'
}
self.assertEqual(expected, json.loads(event.as_json()))
class TestCase(unittest.TestCase):
def __init__(self, *args):
unittest.TestCase.__init__(self, *args)
self.maxDiff = 4096
self.normalizer = LogNormalizer()
self._load_stock_rules()
def _load_stock_rules(self):
"""
Loads all rules db files within the default rules location. When
run with nosetests, this should be the rules directory under the
pylognunit project root. All rules db files must be text files that
end with the suffix '.db'
"""
if os.path.isdir(RULES_LOCATION):
for possible_rule_db in os.listdir(RULES_LOCATION):
if possible_rule_db.endswith('.db'):
self.normalizer.load_rules(
os.path.join(RULES_LOCATION, possible_rule_db))
def load_rule(self, rule):
"""
Loads a single rule into the normalizer context for this test
class. This method is only available if the liblognorm library
version is greater than or equal to 0.3.7
"""
self.normalizer.load_rule(rule)
def load_rules_file(self, rule_filename):
"""
Loads a file of rules into the normalizer context for this test
class. This method is available to all versions of liblognorm.
"""
if not os.path.exists(rule_filename):
raise IOError('Unable to locate rules file: {}'.format(
os.path.join(os.getcwd(), rule_filename)))
self.normalizer.load_rules(rule_filename)
def normalize(self, source):
"""
Attempts to normalize the given source string. This source may be
a PyString, a byte array or a unicode object.
"""
event = self.normalizer.normalize(source)
return json.loads(event.as_json())
def test_loading_rules(self):
normalizer = LogNormalizer()
normalizer.load_rules('./samples/rules.db')
def __init__(self, *args):
unittest.TestCase.__init__(self, *args)
self.maxDiff = 4096
self.normalizer = LogNormalizer()
self._load_stock_rules()
def test_init(self):
normalizer = LogNormalizer()
def test_loading_rules(self):
normalizer = LogNormalizer()
normalizer.load_rules('./samples/rules.db')