def test_query(self):
from collections import OrderedDict as od
cases = [
('', ''),
('', []),
('', {}),
('?name', 'name'),
('?name', [('name', None)]),
('?name', {'name': None}),
('?name=foo', 'name=foo'),
('?name=foo', [('name', 'foo')]),
('?name=foo', {'name': 'foo'}),
('?name=foo', {'name': ['foo']}),
('?name=42', [('name', 42)]),
('?name=42', {'name': 42}),
('?name=42', {'name': [42]}),
('?name=foo&type=bar', [('name', 'foo'), ('type', 'bar')]),
('?name=foo&type=bar', od([('name', 'foo'), ('type', 'bar')])),
('?name=foo&name=bar', [('name', 'foo'), ('name', 'bar')]),
('?name=foo&name=bar', {'name': ['foo', 'bar']}),
('?name=a%2Fb%2Fc', dict(name='a/b/c')),
('?name=a%3Ab%3Ac', dict(name='a:b:c')),
('?name=a%3Fb%3Fc', dict(name='a?b?c')),
('?name=a%40b%40c', dict(name='a@b@c')),
('?name=a%23b%23c', dict(name='a#b#c')),
('?name=a%26b%26c', dict(name='a&b&c')),
('?name=a%3Bb%3Bc', dict(name='a;b;c')),
]
for uri, query in cases:
self.check(uri, query=query)
# invalid query type
for query in (0, [1]):
with self.assertRaises(TypeError, msg='query=%r' % query):
uricompose(query=query)
def test_scheme(self):
cases = [
('foo+bar:', 'foo+bar'),
('foo+bar:', 'FOO+BAR'),
]
for uri, scheme in cases:
self.check(uri, scheme=scheme)
# invalid scheme
for scheme in ('foo:', '\xf6lk\xfcrbis'):
with self.assertRaises(ValueError, msg='scheme=%r' % scheme):
uricompose(scheme=scheme)
def can_fetch(self, useragent, url):
if isinstance(url, bytes):
SLASH = b'/'
Q = b'?'
else:
SLASH = '/'
Q= '?'
splitted = urilib.urisplit(url)
userinfo = splitted.getuserinfo()
if userinfo:
userinfo = _unicodenormalize(userinfo)
urlbase = urilib.uricompose(
scheme=splitted.getscheme(),
userinfo=userinfo,
host=splitted.gethost(),
port=splitted.getport(),
path=SLASH)
if urlbase != SLASH and self.base != urlbase:
# don't have to do with this
return False
target = self.cached.get(useragent, None)
if not target:
for ruleset in self.rulesets:
if ruleset.does_user_agent_match(useragent):
target = ruleset
self.cached[useragent] = target
break
if useragent != '*' and not target:
target = self.default
if target:
path = splitted.getpath()
if path:
path = _unicodenormalize(path)
query = splitted.query
if query:
qsl = urilib.querylist(_unicodenormalize(query))
else:
qsl = None
norm = urilib.uricompose(path=path, query=qsl)
if url[-1] == Q:
norm = norm + Q
return target.is_url_allowed(norm, self.method)
return True
def test_path(self):
cases = [
('foo', 'foo'),
('foo+bar', 'foo+bar'),
('foo%20bar', 'foo bar'),
('/this:that', 'this:that'),
('/this:that/', 'this:that/'),
]
for uri, path in cases:
self.check(uri, path=path)
# invalid path with authority
for path in ('foo'):
with self.assertRaises(ValueError, msg='path=%r' % path):
uricompose(authority='auth', path=path)
# invalid path without authority
for path in ('//', '//foo'):
with self.assertRaises(ValueError, msg='path=%r' % path):
uricompose(path=path)
def test_authority(self):
cases = [
('', None),
('', ''),
('//example.com', 'example.com'),
('//example.com', 'example.com:'),
('//[email protected]', '*****@*****.**'),
('//example.com:42', 'example.com:42'),
('//[email protected]:42', '[email protected]:42'),
('//[email protected]:42', '[email protected]:42'),
('//user@[::1]:42', 'user@[::1]:42'),
('//user:[email protected]', 'user:[email protected]'),
]
for uri, authority in cases:
self.check(uri, authority=authority)
# invalid authority type
for authority in (True, 42, 3.14, ipaddress.IPv6Address(u'::1')):
with self.assertRaises(TypeError, msg='authority=%r' % authority):
uricompose(authority=authority)
def check(self, uri, **kwargs):
result = uricompose(**kwargs)
self.assertEqual(uri, result, msg='%r != %r (kwargs=%r)' % (
uri, result, kwargs)
)
def test_authority_kwargs(self):
from ipaddress import IPv4Address, IPv6Address
cases = [
('', [None, None, None]),
('', [None, '', None]),
('//example.com', [None, 'example.com', None]),
('//example.com', [None, 'example.com', '']),
('//[email protected]', ['user', 'example.com', None]),
('//example.com:42', [None, 'example.com', '42']),
('//example.com:42', [None, 'example.com', 42]),
('//[email protected]:42', ['user', 'example.com', '42']),
('//[email protected]:42', ['user', 'example.com', 42]),
('//[email protected]:42', ['user', '127.0.0.1', 42]),
('//[email protected]:42', ['user', IPv4Address(u'127.0.0.1'), 42]),
('//user@[::1]:42', ['user', '::1', 42]),
('//user@[::1]:42', ['user', '[::1]', 42]),
('//user@[::1]:42', ['user', IPv6Address(u'::1'), 42]),
]
for uri, authority in cases:
self.check(uri, authority=authority)
userinfo, host, port = authority
self.check(uri, userinfo=userinfo, host=host, port=port)
# invalid authority value
for authority in ([], ['foo'], ['foo', 'bar'], range(4)):
with self.assertRaises(ValueError, msg='authority=%r' % authority):
uricompose(authority=authority)
# invalid host type
for host in (True, 42, 3.14, ipaddress.IPv6Network(u'2001:db00::0/24')):
with self.assertRaises(AttributeError, msg='host=%r' % host):
uricompose(authority=[None, host, None])
with self.assertRaises(AttributeError, msg='host=%r' % host):
uricompose(host=host)
# invalid host ip-literal
for host in ('[foo]', '[v1.x]'):
with self.assertRaises(ValueError, msg='host=%r' % host):
uricompose(authority=[None, host, None])
with self.assertRaises(ValueError, msg='host=%r' % host):
uricompose(host=host)
# invalid port value
for port in (-1, 'foo', 3.14):
with self.assertRaises(ValueError, msg='port=%r' % port):
uricompose(authority=[None, '', port])
with self.assertRaises(ValueError, msg='port=%r' % port):
uricompose(port=port)