def rows():
return [
Row(
datatypes.datetime('2002-06-24 16:40:23'),
datatypes.address('172.224.24.114'),
'POST',
datatypes.url('/Default.htm'),
0.67,
200,
7930,
),
Row(
datatypes.datetime('2002-05-02 20:18:01'),
datatypes.address('172.22.255.255'),
'GET',
datatypes.url('/images/picture.jpg'),
0.1,
302,
16328,
),
Row(
datatypes.datetime('2002-05-29 12:34:56'),
datatypes.address('9.180.235.203'),
'HEAD',
datatypes.url('/images/picture.jpg'),
0.1,
202,
None,
),
]
def test_address_geoip_isp():
with mock.patch('lars.geoip._GEOIP_IPV4_ISP') as mock_db:
mock_db.org_by_addr.return_value = 'Internet 404'
assert dt.address('127.0.0.1').isp == 'Internet 404'
with mock.patch('lars.geoip._GEOIP_IPV6_ISP') as mock_db:
mock_db.org_by_addr.return_value = 'Internet 404'
assert dt.address('::1').isp == 'Internet 404'
def test_address_geoip_countries():
with mock.patch('lars.geoip._GEOIP_IPV4_GEO') as mock_db:
mock_db.country_code_by_addr.return_value = 'AA'
assert dt.address('127.0.0.1').country == 'AA'
with mock.patch('lars.geoip._GEOIP_IPV6_GEO') as mock_db:
mock_db.country_code_by_addr.return_value = 'BB'
assert dt.address('::1').country == 'BB'
def test_address_geoip_isp():
with mock.patch('lars.geoip._GEOIP_IPV4_ISP') as mock_db:
mock_db.org_by_addr.return_value = 'Internet 404'
assert dt.address('127.0.0.1').isp == 'Internet 404'
with mock.patch('lars.geoip._GEOIP_IPV6_ISP') as mock_db:
mock_db.org_by_addr.return_value = 'Internet 404'
assert dt.address('::1').isp == 'Internet 404'
def test_address_geoip_org():
with mock.patch('lars.geoip._GEOIP_IPV4_ORG') as mock_db:
mock_db.org_by_addr.return_value = 'Acme Inc.'
assert dt.address('127.0.0.1').org == 'Acme Inc.'
with mock.patch('lars.geoip._GEOIP_IPV6_ORG') as mock_db:
mock_db.org_by_addr.return_value = 'Acme Inc.'
assert dt.address('::1').org == 'Acme Inc.'
def test_address_geoip_countries():
with mock.patch('lars.geoip._GEOIP_IPV4_GEO') as mock_db:
mock_db.country_code_by_addr.return_value = 'AA'
assert dt.address('127.0.0.1').country == 'AA'
with mock.patch('lars.geoip._GEOIP_IPV6_GEO') as mock_db:
mock_db.country_code_by_addr.return_value = 'BB'
assert dt.address('::1').country == 'BB'
def test_address_geoip_org():
with mock.patch('lars.geoip._GEOIP_IPV4_ORG') as mock_db:
mock_db.org_by_addr.return_value = 'Acme Inc.'
assert dt.address('127.0.0.1').org == 'Acme Inc.'
with mock.patch('lars.geoip._GEOIP_IPV6_ORG') as mock_db:
mock_db.org_by_addr.return_value = 'Acme Inc.'
assert dt.address('::1').org == 'Acme Inc.'
def rows():
# Construct some test rows with appropriate namedtuples
Row = namedtuple('Row', (
'timestamp', 'client', 'method', 'url', 'time_taken', 'status',
'size',
))
return [
Row(
datatypes.datetime('2002-06-24 16:40:23'),
datatypes.address('172.224.24.114'),
'POST',
datatypes.url('/Default.htm'),
0.67,
200,
7930,
),
Row(
datatypes.datetime('2002-05-02 20:18:01'),
datatypes.address('172.22.255.255'),
'GET',
datatypes.url('/images/picture.jpg'),
0.1,
302,
16328,
),
Row(
datatypes.datetime('2002-05-29 12:34:56'),
datatypes.address('9.180.235.203'),
'HEAD',
datatypes.url('/images/picture.jpg'),
0.1,
202,
None,
),
]
def rows_null_first():
return [
Row(
datatypes.datetime('2002-06-24 16:40:23'),
datatypes.address('172.224.24.114'),
None,
None,
0.01,
408,
0,
),
Row(
datatypes.datetime('2002-05-02 20:18:01'),
datatypes.address('172.22.255.255'),
'GET',
datatypes.url('/images/picture.jpg'),
0.1,
302,
16328,
),
Row(
datatypes.datetime('2002-05-29 12:34:56'),
datatypes.address('9.180.235.203'),
'HEAD',
datatypes.url('/images/picture.jpg'),
0.1,
202,
None,
),
]
def test_address_ipv6():
assert dt.address('::1') == dt.IPv6Address('::1')
assert dt.address('[::1]') == dt.IPv6Port('::1')
assert dt.address('[::1]:80') == dt.IPv6Port('[::1]:80')
assert dt.address('2001:0db8:85a3:0000:0000:8a2e:0370:7334') == dt.IPv6Address('2001:db8:85a3::8a2e:370:7334')
assert dt.address('[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:22') == dt.IPv6Port('[2001:db8:85a3::8a2e:370:7334]:22')
assert dt.address('[fe80::7334]:22') == dt.IPv6Port('[fe80::7334]:22')
with pytest.raises(ValueError):
dt.address('[::1]:100000')
def test_address_ipv4():
assert dt.address('127.0.0.1') == dt.IPv4Address('127.0.0.1')
assert dt.address(b'127.0.0.1:80') == dt.IPv4Port('127.0.0.1:80')
with pytest.raises(ValueError):
dt.address('abc')
with pytest.raises(ValueError):
dt.address('google.com')
with pytest.raises(ValueError):
dt.address('127.0.0.1:100000')
def test_address_ipv4():
assert dt.address('127.0.0.1') == dt.IPv4Address('127.0.0.1')
assert dt.address(b'127.0.0.1:80') == dt.IPv4Port('127.0.0.1:80')
with pytest.raises(ValueError):
dt.address('abc')
with pytest.raises(ValueError):
dt.address('google.com')
with pytest.raises(ValueError):
dt.address('127.0.0.1:100000')
def address_parse(s):
"""
Parse an IPv4 or IPv6 address (and optional port) in a log file.
:param str s: The string containing the address to parse
:returns: A :class:`~lars.datatypes.IPv4Address` value
"""
return dt.address(s) if s != '-' else None
def test_address_ipv6():
assert dt.address('::1') == dt.IPv6Address('::1')
assert dt.address('[::1]') == dt.IPv6Port('::1')
assert dt.address('[::1]:80') == dt.IPv6Port('[::1]:80')
assert dt.address('2001:0db8:85a3:0000:0000:8a2e:0370:7334'
) == dt.IPv6Address('2001:db8:85a3::8a2e:370:7334')
assert dt.address('[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:22'
) == dt.IPv6Port('[2001:db8:85a3::8a2e:370:7334]:22')
assert dt.address('[fe80::7334]:22') == dt.IPv6Port('[fe80::7334]:22')
with pytest.raises(ValueError):
dt.address('[::1]:100000')
def test_address_geoip_cities():
with mock.patch('lars.geoip._GEOIP_IPV4_GEO') as mock_db:
mock_db.region_by_addr.return_value = {'region_name': 'AA'}
assert dt.address('127.0.0.1').region == 'AA'
mock_db.record_by_addr.return_value = {'city': 'Timbuktu'}
assert dt.address('127.0.0.1').city == 'Timbuktu'
mock_db.record_by_addr.return_value = {'longitude': 1, 'latitude': 2}
assert dt.address('127.0.0.1').coords == geoip.GeoCoord(1, 2)
mock_db.record_by_addr.return_value = None
assert dt.address('127.0.0.1').city is None
assert dt.address('127.0.0.1').coords is None
with mock.patch('lars.geoip._GEOIP_IPV6_GEO') as mock_db:
mock_db.region_by_addr.return_value = {'region_name': 'BB'}
assert dt.address('::1').region == 'BB'
mock_db.record_by_addr.return_value = {'city': 'Transylvania'}
assert dt.address('::1').city == 'Transylvania'
mock_db.record_by_addr.return_value = {'longitude': 3, 'latitude': 4}
assert dt.address('::1').coords == geoip.GeoCoord(3, 4)
mock_db.record_by_addr.return_value = None
assert dt.address('::1').city is None
assert dt.address('::1').coords is None
def test_address_geoip_cities():
with mock.patch('lars.geoip._GEOIP_IPV4_GEO') as mock_db:
mock_db.region_by_addr.return_value = {'region_name': 'AA'}
assert dt.address('127.0.0.1').region == 'AA'
mock_db.record_by_addr.return_value = {'city': 'Timbuktu'}
assert dt.address('127.0.0.1').city == 'Timbuktu'
mock_db.record_by_addr.return_value = {'longitude': 1, 'latitude': 2}
assert dt.address('127.0.0.1').coords == geoip.GeoCoord(1, 2)
mock_db.record_by_addr.return_value = None
assert dt.address('127.0.0.1').city is None
assert dt.address('127.0.0.1').coords is None
with mock.patch('lars.geoip._GEOIP_IPV6_GEO') as mock_db:
mock_db.region_by_addr.return_value = {'region_name': 'BB'}
assert dt.address('::1').region == 'BB'
mock_db.record_by_addr.return_value = {'city': 'Transylvania'}
assert dt.address('::1').city == 'Transylvania'
mock_db.record_by_addr.return_value = {'longitude': 3, 'latitude': 4}
assert dt.address('::1').coords == geoip.GeoCoord(3, 4)
mock_db.record_by_addr.return_value = None
assert dt.address('::1').city is None
assert dt.address('::1').coords is None
def rows():
# Construct some test rows with appropriate namedtuples
Row = namedtuple('Row', (
'timestamp',
'client',
'method',
'url',
'time_taken',
'status',
'size',
))
return [
Row(
datatypes.datetime('2002-06-24 16:40:23'),
datatypes.address('172.224.24.114'),
'POST',
datatypes.url('/Default.htm'),
0.67,
200,
7930,
),
Row(
datatypes.datetime('2002-05-02 20:18:01'),
datatypes.address('172.22.255.255'),
'GET',
datatypes.url('/images/picture.jpg'),
0.1,
302,
16328,
),
Row(
datatypes.datetime('2002-05-29 12:34:56'),
datatypes.address('9.180.235.203'),
'HEAD',
datatypes.url('/images/picture.jpg'),
0.1,
202,
None,
),
]
def test_resolving():
assert dt.hostname('localhost').address == dt.IPv4Address('127.0.0.1')
# This is a bit of a hack; largely depends on the system that's running the
# tests whether these work or not (localhost can be called all sorts of
# things) but the values below work for vanilla Ubuntu hosts and Travis
# CI's VMs
assert dt.hostname('localhost').address.hostname in (
dt.hostname('localhost'),
dt.hostname('localhost.localdomain'),
)
assert dt.hostname('test.invalid').address is None
assert dt.address('127.0.0.1').hostname in (
dt.hostname('localhost'),
dt.hostname('localhost.localdomain'),
)
def test_resolving():
assert dt.hostname('localhost').address == dt.IPv4Address('127.0.0.1')
# This is a bit of a hack; largely depends on the system that's running the
# tests whether these work or not (localhost can be called all sorts of
# things) but the values below work for vanilla Ubuntu hosts and Travis
# CI's VMs
assert dt.hostname('localhost').address.hostname in (
dt.hostname('localhost'),
dt.hostname('localhost.localdomain'),
)
assert dt.hostname('test.invalid').address is None
assert dt.address('127.0.0.1').hostname in (
dt.hostname('localhost'),
dt.hostname('localhost.localdomain'),
)
def test_url():
assert dt.url('foo') == dt.Url('', '', 'foo', '', '', '')
assert dt.url('//foo/bar') == dt.Url('', 'foo', '/bar', '', '', '')
assert dt.url('http://foo/') == dt.Url('http', 'foo', '/', '', '', '')
assert dt.url('http://foo/bar?baz=quux') == dt.Url('http', 'foo', '/bar', '', 'baz=quux', '')
assert dt.url('https://foo/bar#baz') == dt.Url('https', 'foo', '/bar', '', '', 'baz')
u = dt.url('http://localhost/foo/bar#baz')
assert u.scheme == 'http'
assert u.netloc == 'localhost'
assert u.path == dt.Path('/foo', 'bar', '')
assert u.path_str == '/foo/bar'
assert u.fragment == 'baz'
assert u.username is None
assert u.password is None
assert u.port is None
assert u.hostname == dt.hostname('localhost')
assert u.hostname.address == dt.address('127.0.0.1')
def test_url():
assert dt.url('foo') == dt.Url('', '', 'foo', '', '', '')
assert dt.url('//foo/bar') == dt.Url('', 'foo', '/bar', '', '', '')
assert dt.url('http://foo/') == dt.Url('http', 'foo', '/', '', '', '')
assert dt.url('http://foo/bar?baz=quux') == dt.Url('http', 'foo', '/bar',
'', 'baz=quux', '')
assert dt.url('https://foo/bar#baz') == dt.Url('https', 'foo', '/bar', '',
'', 'baz')
u = dt.url('http://localhost/foo/bar#baz')
assert u.scheme == 'http'
assert u.netloc == 'localhost'
assert u.path == dt.Path('/foo', 'bar', '')
assert u.path_str == '/foo/bar'
assert u.fragment == 'baz'
assert u.username is None
assert u.password is None
assert u.port is None
assert u.hostname == dt.hostname('localhost')
assert u.hostname.address == dt.address('127.0.0.1')
def test_address_port_manipulation():
addr = dt.address('127.0.0.1:80')
assert str(addr) == '127.0.0.1:80'
addr.port = None
assert str(addr) == '127.0.0.1'
def test_address():
with mock.patch('lars.dns.from_address') as from_address:
from_address.return_value = '0.0.0.0'
assert dt.address('0.0.0.0').hostname is None
from_address.return_value = '::'
assert dt.address('::').hostname is None
def test_address():
with mock.patch('lars.dns.from_address') as from_address:
from_address.return_value = '0.0.0.0'
assert dt.address('0.0.0.0').hostname is None
from_address.return_value = '::'
assert dt.address('::').hostname is None
def test_address_port_manipulation():
addr = dt.address('127.0.0.1:80')
assert str(addr) == '127.0.0.1:80'
addr.port = None
assert str(addr) == '127.0.0.1'