Ejemplo n.º 1
0
 def test_nginx_log_match(self):
     text = (
         'edge.v.iask.com.edge.sinastorage.com 14.18.243.65 6.032s - [21/Jul/2014:16:00:02 +0800]'
         ' "GET /edge.v.iask.com/125880034.hlv HTTP/1.0" 200 70528990 "-"'
         ' "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36"'
     )
     pat = (
         '%{HOST:host} %{IP:client_ip} %{NUMBER:delay:float}s - \[%{HTTPDATE:time_stamp}\]'
         ' "%{WORD:verb} %{URIPATHPARAM:uri_path} HTTP/%{NUMBER:http_ver:float}" %{INT:http_status:int} %{INT:bytes:int} %{QS:referrer}'
         ' %{QS:agent}'
     )
     match = grok_match(text, pat)
     self.assertIsNotNone(match)
     self.assertEqual(match['host'], 'edge.v.iask.com.edge.sinastorage.com')
     self.assertEqual(match['client_ip'], '14.18.243.65')
     self.assertEqual(match['delay'], 6.032)
     self.assertEqual(match['time_stamp'], '21/Jul/2014:16:00:02 +0800')
     self.assertEqual(match['verb'], 'GET')
     self.assertEqual(match['uri_path'], '/edge.v.iask.com/125880034.hlv')
     self.assertEqual(match['http_ver'], 1.0)
     self.assertEqual(match['http_status'], 200)
     self.assertEqual(match['bytes'], 70528990)
     self.assertEqual(match['referrer'], '"-"')
     self.assertEqual(match['agent'],
                      '"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36"')
Ejemplo n.º 2
0
 def test_qs(self):
     text = 'gary "25" "never quit" "blah"'
     pat = '%{WORD:name} "%{INT:age}" %{QS:motto} %{QS:blah}'
     match = grok_match(text, pat)
     self.assertIsNotNone(match)
     self.assertEqual(match['name'], 'gary')
     self.assertEqual(match['age'], '25')
     self.assertEqual(match['motto'], '"never quit"')
Ejemplo n.º 3
0
 def test_multiple_patterns(self):
     text = 'gary 25 "never quit"'
     pat = '%{WORD:name} %{INT:age} %{QUOTEDSTRING:motto}'
     match = grok_match(text, pat)
     self.assertIsNotNone(match)
     self.assertEqual(match['name'], 'gary')
     self.assertEqual(match['age'], '25')
     self.assertEqual(match['motto'], '"never quit"')
Ejemplo n.º 4
0
 def test_syslog_match(self):
     text = (
         '2016-03-25T05:55:32 <3245521.83838> host kernel: [1858.738417]  [<ffffffff811e6c31>] SyS_ioctl+0x81/0xa0'
     )
     pat = '%{SYSLOGLINE}'
     match = grok_match(text, pat, auto_map=True)
     self.assertEqual(match['facility'], 3245521)
     self.assertEqual(match['pid'], None)
     self.assertEqual(match['priority'], 83838)
Ejemplo n.º 5
0
 def test_custom_pat_files(self):
     pats_dir = os.path.join(os.path.dirname(__file__), 'test_patterns')
     text = 'Beijing-1104,gary 25 "never quit"'
     pat = '%{ID:user_id},%{WORD:name} %{INT:age} %{QUOTEDSTRING:motto}'
     match = grok_match(text, pat, custom_patterns_dir=pats_dir)
     self.assertIsNotNone(match)
     self.assertEqual(match['user_id'], 'Beijing-1104')
     self.assertEqual(match['name'], 'gary')
     self.assertEqual(match['age'], '25')
     self.assertEqual(match['motto'], '"never quit"')
Ejemplo n.º 6
0
 def test_custom_patterns(self):
     custom_pats = {'ID' : '%{WORD}-%{INT}'}
     text = 'Beijing-1104,gary 25 "never quit"'
     pat = '%{ID:user_id},%{WORD:name} %{INT:age} %{QUOTEDSTRING:motto}'
     match = grok_match(text, pat, custom_patterns=custom_pats)
     self.assertIsNotNone(match)
     self.assertEqual(match['user_id'], 'Beijing-1104')
     self.assertEqual(match['name'], 'gary')
     self.assertEqual(match['age'], '25')
     self.assertEqual(match['motto'], '"never quit"')
Ejemplo n.º 7
0
 def test_syslog_match(self):
     pats_dir = os.path.join(os.path.dirname(__file__), 'test_patterns')
     text = (
         '2016-03-25T05:55:32 <3245521.83838> host kernel: [1858.738417]  [<ffffffff811e6c31>] SyS_ioctl+0x81/0xa0'
     )
     pat = '%{TYPEDSYSLOGLINE}'
     match = grok_match(text, pat, custom_patterns_dir=pats_dir)
     self.assertEqual(match['facility'], 3245521)
     self.assertEqual(match['pid'], None)
     self.assertEqual(match['priority'], 83838)
Ejemplo n.º 8
0
    def test_missing_variable_name(self):
        '''
        test_missing_variable_name

        You get empty dict because variable name is not set,
        compare "%{WORD}" and "%{WORD:variable_name}"
        '''
        text = 'github'
        pat = '%{WORD}'
        match = grok_match(text, pat)
        self.assertIsNotNone(match)
        self.assertEqual(match, {})
Ejemplo n.º 9
0
 def test_compound_matches(self):
     text = (
         '127.0.0.1 - - [15/Sep/2015:13:41:35 -0400] "GET /index.html '
         'HTTP/1.1" 502 352 "-" '
         '"Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0"'
     )
     pat = '%{COMBINEDAPACHELOG}'
     match = grok_match(text, pat, auto_map=True)
     self.assertIsNotNone(match)
     self.assertEqual(match['clientip'], '127.0.0.1')
     self.assertEqual(match['ident'], '-')
     self.assertEqual(match['auth'], '-')
     self.assertEqual(match['timestamp'], '15/Sep/2015:13:41:35 -0400')
     self.assertEqual(match['verb'], 'GET')
     self.assertEqual(match['request'], '/index.html')
     self.assertEqual(match['httpversion'], 1.1)
     self.assertEqual(match['response'], 502)
     self.assertEqual(match['bytes'], 352)
     self.assertEqual(match['referrer'], '"-"')
     self.assertEqual(match['agent'], '"Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0"')
Ejemplo n.º 10
0
 def test_compound_matches(self):
     pats_dir = os.path.join(os.path.dirname(__file__), 'test_patterns')
     text = (
         '127.0.0.1 - - [15/Sep/2015:13:41:35 -0400] "GET /index.html '
         'HTTP/1.1" 502 352 "-" '
         '"Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0"'
     )
     pat = '%{TYPEDCOMBINEDAPACHELOG}'
     match = grok_match(text, pat, custom_patterns_dir=pats_dir)
     self.assertIsNotNone(match)
     self.assertEqual(match['clientip'], '127.0.0.1')
     self.assertEqual(match['ident'], '-')
     self.assertEqual(match['auth'], '-')
     self.assertEqual(match['timestamp'], '15/Sep/2015:13:41:35 -0400')
     self.assertEqual(match['verb'], 'GET')
     self.assertEqual(match['request'], '/index.html')
     self.assertEqual(match['httpversion'], 1.1)
     self.assertEqual(match['response'], 502)
     self.assertEqual(match['bytes'], 352)
     self.assertEqual(match['referrer'], '"-"')
     self.assertEqual(match['agent'], '"Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0"')
Ejemplo n.º 11
0
 def test_non_neg_int_match(self):
     text = '3245521'
     pat = '%{NONNEGINT:test_int:int}'
     match = grok_match(text, pat)
     self.assertIsNotNone(match)
     self.assertEqual(match['test_int'], 3245521)
Ejemplo n.º 12
0
 def test_int_match(self):
     text = '1024'
     pat = '%{INT:test_int}'
     match = grok_match(text, pat)
     self.assertIsNotNone(match)
     self.assertEqual(match['test_int'], '1024')
Ejemplo n.º 13
0
 def test_no_match(self):
     text = 'github'
     pat = '%{NUMBER:test_num}'
     match = grok_match(text, pat)
     self.assertIsNone(match)
Ejemplo n.º 14
0
 def test_host_match(self):
     text = 'github.com'
     pat = '%{HOST:website}'
     match = grok_match(text, pat)
     self.assertIsNotNone(match)
     self.assertEqual(match['website'], text.strip())
Ejemplo n.º 15
0
 def test_timestamp_iso8601_match(self):
     text = '1989-11-04 05:33:02+0800'
     pat = '%{TIMESTAMP_ISO8601:ts}'
     match = grok_match(text, pat)
     self.assertIsNotNone(match)
     self.assertEqual(match['ts'], text.strip())
Ejemplo n.º 16
0
 def test_ip_match(self):
     text = '192.168.1.1'
     pat = '%{IP:ip}'
     match = grok_match(text, pat)
     self.assertIsNotNone(match)
     self.assertEqual(match['ip'], text.strip())
Ejemplo n.º 17
0
 def test_number_match(self):
     text = '1024'
     pat = '%{NUMBER:test_num}'
     match = grok_match(text, pat)
     self.assertIsNotNone(match)
     self.assertEqual(match['test_num'], '1024')
Ejemplo n.º 18
0
 def test_word_match(self):
     text = 'garyelephant '
     pat = '%{WORD:name} '
     match = grok_match(text, pat)
     self.assertIsNotNone(match)
     self.assertEqual(match['name'], text.strip())
Ejemplo n.º 19
0
 def test_unsupported_type(self):
     text = 'unknown_type'
     pat = '%{WORD:test_str:decimal}'
     match = grok_match(text, pat)
     self.assertIsNotNone(match)
     self.assertEqual(match['test_str'], 'unknown_type')
Ejemplo n.º 20
0
 def test_missing_variable_names(self):
     text = 'gary 25 "never quit"'
     pat = '%{WORD} %{INT} %{QUOTEDSTRING}'
     match = grok_match(text, pat)
     self.assertIsNotNone(match)
     self.assertEqual(match, {})
Ejemplo n.º 21
0
 def test_not_match(self):
     #"male" is not INT
     text = 'gary male "never quit"'
     pat = '%{WORD:name} %{INT:age} %{QUOTEDSTRING:motto}'
     match = grok_match(text, pat)
     self.assertIsNone(match)
Ejemplo n.º 22
0
 def test_conversion_failure_float(self):
     text = 'not_a_float'
     pat = '%{WORD:test_str:float}'
     match = grok_match(text, pat)
     self.assertIsNotNone(match)
     self.assertEqual(match['test_str'], 'not_a_float')