def common_jog():
"""
Common Log Format
"""
def static_reader():
return """
127.0.0.1 abc frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 404 82
asdasd
127.0.0.1 abc susie [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
127.0.0.2 ghi frank [12/Oct/2000:13:55:36 -0700] "GET /apache_pb.jpg HTTP/1.0" 200 1326
"""
import datetime
regex = '([(\d\.)]+) (.*) (.*) \[(.*?)\] "(.*?)" (\d+) (\d+)'
strptime = datetime.datetime.strptime
value = lambda x: x
int_value = lambda x: int(x)
date_value = lambda x: strptime(x.split(' ')[0], '%d/%b/%Y:%H:%M:%S')
field_map = [('address', value), ('identifier', value), ('userid', value),
('timestamp', date_value), ('request', value),
('response_code', int_value), ('response_size', int_value)]
parser = partial(regex_parser, regex, field_map)
class Log(object):
address = identifier = userid = request = 'UNPARSED'
response_code = -1
# response_size = -1
timestamp = datetime.datetime(year=datetime.MINYEAR, month=1, day=1)
def tests():
test_log = _Jogger(reader=static_reader, parser=parser, log=Log).jog()
assert len(test_log) == 4
assert len(test_log(lambda line: line.unparsed)) == 1
line = test_log[0]
assert line.address == "127.0.0.1"
assert line.identifier == 'abc'
assert line.userid == 'frank'
assert line.timestamp == datetime.datetime(2000, 10, 10, 13, 55, 36)
assert line.request == "GET /apache_pb.gif HTTP/1.0"
assert line.response_code == 404
assert line.response_size == 82
return (_Jogger(parser=parser, log=Log).jog, tests)
def tests():
test_log = _Jogger(reader=static_reader, parser=parser, log=Log).jog()
assert len(test_log) == 4
assert len(test_log(lambda line: line.unparsed)) == 1
line = test_log[0]
assert line.address == "127.0.0.1"
assert line.identifier == 'abc'
assert line.userid == 'frank'
assert line.timestamp == datetime.datetime(2000, 10, 10, 13, 55, 36)
assert line.request == "GET /apache_pb.gif HTTP/1.0"
assert line.response_code == 404
assert line.response_size == 82
def tests(): test_log = _Jogger(reader=static_reader, parser=parser, log=Log).jog() assert len(test_log) == 4 assert len(test_log(lambda line: line.unparsed)) == 1 line = test_log[0] assert line.address == "127.0.0.1" assert line.identifier == 'abc' assert line.userid == 'frank' assert line.timestamp == datetime.datetime(2000, 10, 10, 13, 55, 36) assert line.request == "GET /apache_pb.gif HTTP/1.0" assert line.response_code == 404 assert line.response_size == 82
def tests():
test_log = _Jogger(reader=static_reader, parser=parser, log=Log).jog()
assert len(test_log) == 4
assert len(test_log(lambda line: line.unparsed)) == 1
line = test_log[0]
assert line.address == "123.65.150.10"
assert line.identifier == '-'
assert line.userid == '-'
assert line.timestamp == datetime.datetime(2010, 8, 23, 3, 50, 59)
assert line.request == "POST /wordpress3/wp-admin/admin-ajax.php HTTP/1.1"
assert line.response_code == 200
assert line.response_size == 2
assert line.referer == "http://www.example.com/wordpress3/wp-admin/post-new.php"
assert line.user_agent == "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.25 Safari/534.3"
def tests(): test_log = _Jogger(reader=static_reader, parser=parser, log=Log).jog() assert len(test_log) == 4 assert len(test_log(lambda line: line.unparsed)) == 1 line = test_log[0] assert line.address == "123.65.150.10" assert line.identifier == '-' assert line.userid == '-' assert line.timestamp == datetime.datetime(2010, 8, 23, 3, 50, 59) assert line.request == "POST /wordpress3/wp-admin/admin-ajax.php HTTP/1.1" assert line.response_code == 200 assert line.response_size == 2 assert line.referer == "http://www.example.com/wordpress3/wp-admin/post-new.php" assert line.user_agent == "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.25 Safari/534.3"
def combined_jog():
"""
Combined Log Format
"""
def static_reader():
return """
123.65.150.10 - - [23/Aug/2010:03:50:59 +0000] "POST /wordpress3/wp-admin/admin-ajax.php HTTP/1.1" 200 2 "http://www.example.com/wordpress3/wp-admin/post-new.php" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.25 Safari/534.3"
asdasd
123.65.150.10 - - [23/Aug/2010:03:50:59 +0000] "POST /wordpress3/wp-admin/admin-ajax.php HTTP/1.1" 200 2 "http://www.example.com/wordpress3/wp-admin/post-new.php" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.25 Safari/534.3"
123.65.150.10 - - [23/Aug/2010:03:50:59 +0000] "POST /wordpress3/wp-admin/admin-ajax.php HTTP/1.1" 200 2 "http://www.example.com/wordpress3/wp-admin/post-new.php" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.25 Safari/534.3"
"""
import datetime
regex = '([(\d\.)]+) (.*) (.*) \[(.*?)\] "(.*?)" (\d+) (\d+) "(.*?)" "(.*?)"'
strptime = datetime.datetime.strptime
value = lambda x: x
int_value = lambda x: int(x)
date_value = lambda x: strptime(x.split(' ')[0], '%d/%b/%Y:%H:%M:%S')
field_map = [
('address', value),
('identifier', value),
('userid', value),
('timestamp', date_value),
('request', value),
('response_code', int_value),
('response_size', int_value),
('referer', value),
('user_agent', value),
]
parser = partial(regex_parser, regex, field_map)
class Log(object):
address = identifier = userid = request = referer = user_agent = 'UNPARSED'
response_code = -1
response_size = -1
timestamp = datetime.datetime(year=datetime.MINYEAR, month=1, day=1)
def tests():
test_log = _Jogger(reader=static_reader, parser=parser, log=Log).jog()
assert len(test_log) == 4
assert len(test_log(lambda line: line.unparsed)) == 1
line = test_log[0]
assert line.address == "123.65.150.10"
assert line.identifier == '-'
assert line.userid == '-'
assert line.timestamp == datetime.datetime(2010, 8, 23, 3, 50, 59)
assert line.request == "POST /wordpress3/wp-admin/admin-ajax.php HTTP/1.1"
assert line.response_code == 200
assert line.response_size == 2
assert line.referer == "http://www.example.com/wordpress3/wp-admin/post-new.php"
assert line.user_agent == "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.25 Safari/534.3"
return (_Jogger(parser=parser, log=Log).jog, tests)
def common_jog():
"""
Common Log Format
"""
def static_reader():
return """
127.0.0.1 abc frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 404 82
asdasd
127.0.0.1 abc susie [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
127.0.0.2 ghi frank [12/Oct/2000:13:55:36 -0700] "GET /apache_pb.jpg HTTP/1.0" 200 1326
"""
import datetime
regex = '([(\d\.)]+) (.*) (.*) \[(.*?)\] "(.*?)" (\d+) (\d+)'
strptime = datetime.datetime.strptime
value = lambda x: x
int_value = lambda x: int(x)
date_value = lambda x: strptime(x.split(' ')[0], '%d/%b/%Y:%H:%M:%S')
field_map = [
('address', value),
('identifier', value),
('userid', value),
('timestamp', date_value),
('request', value),
('response_code', int_value),
('response_size', int_value)
]
parser = partial(regex_parser, regex, field_map)
class Log(object):
address = identifier = userid = request = 'UNPARSED'
response_code = -1
# response_size = -1
timestamp = datetime.datetime(year=datetime.MINYEAR, month=1, day=1)
def tests():
test_log = _Jogger(reader=static_reader, parser=parser, log=Log).jog()
assert len(test_log) == 4
assert len(test_log(lambda line: line.unparsed)) == 1
line = test_log[0]
assert line.address == "127.0.0.1"
assert line.identifier == 'abc'
assert line.userid == 'frank'
assert line.timestamp == datetime.datetime(2000, 10, 10, 13, 55, 36)
assert line.request == "GET /apache_pb.gif HTTP/1.0"
assert line.response_code == 404
assert line.response_size == 82
return (_Jogger(parser=parser, log=Log).jog, tests)
def combined_jog():
"""
Combined Log Format
"""
def static_reader():
return """
123.65.150.10 - - [23/Aug/2010:03:50:59 +0000] "POST /wordpress3/wp-admin/admin-ajax.php HTTP/1.1" 200 2 "http://www.example.com/wordpress3/wp-admin/post-new.php" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.25 Safari/534.3"
asdasd
123.65.150.10 - - [23/Aug/2010:03:50:59 +0000] "POST /wordpress3/wp-admin/admin-ajax.php HTTP/1.1" 200 2 "http://www.example.com/wordpress3/wp-admin/post-new.php" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.25 Safari/534.3"
123.65.150.10 - - [23/Aug/2010:03:50:59 +0000] "POST /wordpress3/wp-admin/admin-ajax.php HTTP/1.1" 200 2 "http://www.example.com/wordpress3/wp-admin/post-new.php" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.25 Safari/534.3"
"""
import datetime
regex = '([(\d\.)]+) (.*) (.*) \[(.*?)\] "(.*?)" (\d+) (\d+) "(.*?)" "(.*?)"'
strptime = datetime.datetime.strptime
value = lambda x: x
int_value = lambda x: int(x)
date_value = lambda x: strptime(x.split(' ')[0], '%d/%b/%Y:%H:%M:%S')
field_map = [
('address', value),
('identifier', value),
('userid', value),
('timestamp', date_value),
('request', value),
('response_code', int_value),
('response_size', int_value),
('referer', value),
('user_agent', value),
]
parser = partial(regex_parser, regex, field_map)
class Log(object):
address = identifier = userid = request = referer = user_agent = 'UNPARSED'
response_code = -1
response_size = -1
timestamp = datetime.datetime(year=datetime.MINYEAR, month=1, day=1)
def tests():
test_log = _Jogger(reader=static_reader, parser=parser, log=Log).jog()
assert len(test_log) == 4
assert len(test_log(lambda line: line.unparsed)) == 1
line = test_log[0]
assert line.address == "123.65.150.10"
assert line.identifier == '-'
assert line.userid == '-'
assert line.timestamp == datetime.datetime(2010, 8, 23, 3, 50, 59)
assert line.request == "POST /wordpress3/wp-admin/admin-ajax.php HTTP/1.1"
assert line.response_code == 200
assert line.response_size == 2
assert line.referer == "http://www.example.com/wordpress3/wp-admin/post-new.php"
assert line.user_agent == "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.25 Safari/534.3"
return (_Jogger(parser=parser, log=Log).jog, tests)
