def test_normal_re(self): #A pattern without oniguruma patterns should work regexp = "My name is (?P<name>\w*) and I'm (?P<age>\d*) years old\." compiled = regex.compile(pattern_matching.grok_re_preprocess(regexp)) self.assertDictEqual( compiled.match("My name is Takeshi and I'm 4 years old.").groupdict(), {"name": "Takeshi", "age": '4'})
def test_nginx_log(self): regexp = '<%{INT}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:host} %{IPORHOST:remote_addr} - %{USERNAME:remote_user}?- \[%{HTTPDATE:time_local}\] \"(?P<method>(GET|PUT|PATCH|POST|DELETE|HEAD|OPTIONS)) %{URIPATH:path}%{URIPARAM:params} HTTP/%{NUMBER:httpversion}\" %{INT:status} %{INT:body_bytes_sent}\"-\" %{QS:http_user_agent}' other_log = '<13>Mar 25 12:26:57 myserver.io 62.73.84.230 - - [25/Mar/2016:12:26:57 +0000] "GET /orders?order_identifier=AB075081&consumer_name=&consumer_first_name= HTTP/1.1" 200 1499"-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36"' compiled = regex.compile(pattern_matching.grok_re_preprocess(regexp)) self.assertDictEqual( compiled.match(other_log).groupdict(), { "syslog_timestamp": "Mar 25 12:26:57", "host": 'myserver.io', "remote_addr": '62.73.84.230', "remote_user": None, "time_local": "25/Mar/2016:12:26:57 +0000", "method": "GET", "path": "/orders", "params": "?order_identifier=AB075081&consumer_name=&consumer_first_name=", "httpversion": "1.1", "status": "200", "body_bytes_sent": "1499", "http_user_agent": '"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36"' })
def test_pattern_synonym(self): regexp = "My name is %{USER:username} and I'm %{INT} years old" compiled = regex.compile(pattern_matching.grok_re_preprocess(regexp)) self.assertDictEqual( compiled.match( "My name is Takeshi and I'm 4 years old.").groupdict(), {"username": "******"})
def test_pattern_types(self): regexp = "My name is %{USER:username} and I'm %{INT:age:int} years old" grokked_re, types = pattern_matching.grok_re_preprocess(regexp) self.assertDictEqual(types, {'age': int}) compiled = regex.compile(grokked_re) self.assertDictEqual( compiled.match("My name is Takeshi and I'm 4 years old.").groupdict(), {"username": "******", "age": '4'})
def test_skip_pattern(self): regexp = "My name is %{USERNAME:name} and I'm %{INT} years old" grokked_re, types = pattern_matching.grok_re_preprocess(regexp) self.assertDictEqual(types, {}) compiled = regex.compile(grokked_re) self.assertDictEqual( compiled.match("My name is Takeshi and I'm 4 years old.").groupdict(), {"name": "Takeshi"})
def test_normal_re(self): #A pattern without oniguruma patterns should work regexp = "My name is (?P<name>\w*) and I'm (?P<age>\d*) years old\." grokked_re, types = pattern_matching.grok_re_preprocess(regexp) self.assertDictEqual(types, {}) compiled = regex.compile(grokked_re) self.assertDictEqual( compiled.match("My name is Takeshi and I'm 4 years old.").groupdict(), {"name": "Takeshi", "age": '4'})
def test_embedded_patterns(self): regexp = "%{SYSLOGTIMESTAMP:timestamp}" compiled = regex.compile(pattern_matching.grok_re_preprocess(regexp)) self.assertDictEqual( compiled.match("Mar 23 12:30:20").groupdict(), {"timestamp": "Mar 23 12:30:20"}) self.assertDictEqual( compiled.match("Mar 23 12:30:20").groupdict(), {"timestamp": "Mar 23 12:30:20"})
def test_simple(self): regexp = "My name is %{USERNAME:name} and I'm %{INT:age} years old" compiled = regex.compile(pattern_matching.grok_re_preprocess(regexp)) self.assertDictEqual( compiled.match( "My name is Takeshi and I'm 4 years old.").groupdict(), { "name": "Takeshi", "age": '4' })
def test_embedded_patterns(self): regexp = "%{SYSLOGTIMESTAMP:timestamp}" compiled = regex.compile(pattern_matching.grok_re_preprocess(regexp)) self.assertDictEqual( compiled.match("Mar 23 12:30:20").groupdict(), {"timestamp": "Mar 23 12:30:20"}) self.assertDictEqual( compiled.match("Mar 23 12:30:20").groupdict(), {"timestamp": "Mar 23 12:30:20"})
def test_nginx_log(self): regexp = '<%{INT}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:host} %{IPORHOST:remote_addr} - %{USERNAME:remote_user}?- \[%{HTTPDATE:time_local}\] \"(?P<method>(GET|PUT|PATCH|POST|DELETE|HEAD|OPTIONS)) %{URIPATH:path}%{URIPARAM:params} HTTP/%{NUMBER:httpversion}\" %{INT:status} %{INT:body_bytes_sent}\"-\" %{QS:http_user_agent}' other_log = '<13>Mar 25 12:26:57 myserver.io 62.73.84.230 - - [25/Mar/2016:12:26:57 +0000] "GET /orders?order_identifier=AB075081&consumer_name=&consumer_first_name= HTTP/1.1" 200 1499"-" "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36"' compiled = regex.compile(pattern_matching.grok_re_preprocess(regexp)) self.assertDictEqual( compiled.match(other_log).groupdict(), {"syslog_timestamp": "Mar 25 12:26:57", "host": 'myserver.io', "remote_addr": '62.73.84.230', "remote_user": None, "time_local": "25/Mar/2016:12:26:57 +0000", "method": "GET", "path": "/orders", "params": "?order_identifier=AB075081&consumer_name=&consumer_first_name=", "httpversion": "1.1", "status": "200", "body_bytes_sent": "1499", "http_user_agent": '"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36"' })
def test_pattern_synonym(self): regexp = "My name is %{USER:username} and I'm %{INT} years old" compiled = regex.compile(pattern_matching.grok_re_preprocess(regexp)) self.assertDictEqual( compiled.match("My name is Takeshi and I'm 4 years old.").groupdict(), {"username": "******"})
def test_simple(self): regexp = "My name is %{USERNAME:name} and I'm %{INT:age} years old" compiled = regex.compile(pattern_matching.grok_re_preprocess(regexp)) self.assertDictEqual( compiled.match("My name is Takeshi and I'm 4 years old.").groupdict(), {"name": "Takeshi", "age": '4'})