def fetch(self, uri, headers=None, body=None):
self.count += 1
if self.count == 1:
headers = {
'X-XRDS-Location'.lower(): 'http://unittest/404',
}
return fetchers.HTTPResponse(uri, 200, headers, '')
else:
return fetchers.HTTPResponse(uri, 404)
def test_fetcher(fetcher, exc, server):
def geturl(path):
host, port = server.server_address
return 'http://%s:%s%s' % (host, port, path)
expected_headers = {'content-type': 'text/plain'}
def plain(path, code):
path = '/' + path
expected = fetchers.HTTPResponse(geturl(path), code, expected_headers,
path)
return (path, expected)
expect_success = fetchers.HTTPResponse(geturl('/success'), 200,
expected_headers, '/success')
cases = [
('/success', expect_success),
('/301redirect', expect_success),
('/302redirect', expect_success),
('/303redirect', expect_success),
('/307redirect', expect_success),
plain('notfound', 404),
plain('badreq', 400),
plain('forbidden', 403),
plain('error', 500),
plain('server_error', 503),
]
for path, expected in cases:
fetch_url = geturl(path)
try:
actual = fetcher.fetch(fetch_url)
except (SystemExit, KeyboardInterrupt):
pass
except Exception as e:
raise AssertionError((fetcher, fetch_url, e))
raise
else:
failUnlessResponseExpected(expected, actual, extra=locals())
for err_url in [
geturl('/closed'), 'http://invalid.janrain.com/', 'not:a/url',
'ftp://janrain.com/pub/'
]:
try:
result = fetcher.fetch(err_url)
except (KeyboardInterrupt, SystemExit):
raise
except fetchers.HTTPError:
# This is raised by the Curl fetcher for bad cases
# detected by the fetchers module, but it's a subclass of
# HTTPFetchingError, so we have to catch it explicitly.
assert exc
except fetchers.HTTPFetchingError:
assert not exc, (fetcher, exc, server)
except:
assert exc
else:
assert False, 'An exception was expected for %r (%r)' % (fetcher,
result)
def test_error(self):
# Test error responses - returned as obtained
self.add_response('http://example.cz/error/', 500,
{'Content-Type': 'text/plain'}, 'BODY')
response = self.fetcher.fetch('http://example.cz/error/')
expected = fetchers.HTTPResponse('http://example.cz/error/', 500,
{'Content-Type': 'text/plain'},
'BODY')
assertResponse(expected, response)
def test_redirect(self):
# Test redirect response - a final response comes from another URL.
self.add_response('http://example.cz/success/', 200,
{'Content-Type': 'text/plain'}, 'BODY')
response = self.fetcher.fetch('http://example.cz/redirect/')
expected = fetchers.HTTPResponse('http://example.cz/success/', 200,
{'Content-Type': 'text/plain'},
'BODY')
assertResponse(expected, response)
def test_success(self):
# Test success response
self.add_response('http://example.cz/success/', 200,
{'Content-Type': 'text/plain'}, 'BODY')
response = self.fetcher.fetch('http://example.cz/success/')
expected = fetchers.HTTPResponse('http://example.cz/success/', 200,
{'Content-Type': 'text/plain'},
'BODY')
assertResponse(expected, response)
def test_fetcher(fetcher, exc, server):
def geturl(path):
return 'http://%s:%s%s' % (socket.getfqdn(
server.server_name), server.socket.getsockname()[1], path)
expected_headers = {'content-type': 'text/plain'}
def plain(path, code):
path = '/' + path
expected = fetchers.HTTPResponse(geturl(path), code, expected_headers,
path.encode('utf-8'))
return (path, expected)
expect_success = fetchers.HTTPResponse(geturl('/success'), 200,
expected_headers, b'/success')
cases = [
('/success', expect_success),
('/301redirect', expect_success),
('/302redirect', expect_success),
('/303redirect', expect_success),
('/307redirect', expect_success),
plain('notfound', 404),
plain('badreq', 400),
plain('forbidden', 403),
plain('error', 500),
plain('server_error', 503),
]
for path, expected in cases:
fetch_url = geturl(path)
try:
actual = fetcher.fetch(fetch_url)
except Exception:
print(fetcher, fetch_url)
raise
else:
assertResponse(expected, actual)
for err_url in [
geturl('/closed'), 'http://invalid.janrain.com/', 'not:a/url',
'ftp://janrain.com/pub/'
]:
try:
result = fetcher.fetch(err_url)
except fetchers.HTTPError:
# This is raised by the Curl fetcher for bad cases
# detected by the fetchers module, but it's a subclass of
# HTTPFetchingError, so we have to catch it explicitly.
assert exc
except fetchers.HTTPFetchingError:
assert not exc, (fetcher, exc, server)
except Exception:
assert exc
else:
assert False, 'An exception was expected for %r (%r)' % (fetcher,
result)
def test_connection_error(self):
# Test connection error
self.http_mock.side_effect = urllib2.HTTPError(
'http://example.cz/error/', 500, 'Error message',
{'Content-Type': 'text/plain'}, StringIO('BODY'))
response = self.fetcher.fetch('http://example.cz/error/')
expected = fetchers.HTTPResponse('http://example.cz/error/', 500,
{'Content-Type': 'text/plain'},
'BODY')
assertResponse(expected, response)
def mkResponse(data):
status_mo = status_header_re.match(data)
headers_str, body = data.split('\n\n', 1)
headers = {}
for line in headers_str.split('\n'):
k, v = line.split(':', 1)
k = k.strip().lower()
v = v.strip()
headers[k] = v
status = int(status_mo.group(1))
return fetchers.HTTPResponse(status=status, headers=headers, body=body)
def test_error(self):
# Test error responses - returned as obtained
with responses.RequestsMock() as rsps:
rsps.add(responses.GET,
'http://example.cz/error/',
status=500,
body=b'BODY',
headers={'Content-Type': 'text/plain'})
response = self.fetcher.fetch('http://example.cz/error/')
expected = fetchers.HTTPResponse('http://example.cz/error/', 500,
{'Content-Type': 'text/plain'},
b'BODY')
assertResponse(expected, response)
def test_get(self):
# Test GET response
with responses.RequestsMock() as rsps:
rsps.add(responses.GET,
'http://example.cz/',
status=200,
body=b'BODY',
headers={'Content-Type': 'text/plain'})
response = self.fetcher.fetch('http://example.cz/')
expected = fetchers.HTTPResponse('http://example.cz/', 200,
{'Content-Type': 'text/plain'},
b'BODY')
assertResponse(expected, response)
def fetch(self, url, body=None, headers=None):
"""
This performs an HTTP POST or GET, following redirects along
the way. If a body is specified, then the request will be a
POST. Otherwise, it will be a GET.
@param headers: HTTP headers to include with the request
@type headers: {str:str}
@return: An object representing the server's HTTP response. If
there are network or protocol errors, an exception will be
raised. HTTP error responses, like 404 or 500, do not
cause exceptions.
@rtype: L{HTTPResponse}
@raise Exception: Different implementations will raise
different errors based on the underlying HTTP library.
"""
if not fetchers._allowedURL(url):
raise ValueError('Bad URL scheme: %r' % (url,))
if not headers:
headers = {}
if body:
method = urlfetch.POST
if 'Content-Type' not in headers:
headers['Content-Type'] = 'application/x-www-form-urlencoded'
else:
method = urlfetch.GET
if not headers:
headers = {}
# follow up to 10 redirects
for i in range(10):
resp = urlfetch.fetch(url, body, method, headers)
if resp.status_code in (301, 302):
logging.debug('Following %d redirect to %s' %
(resp.status_code, resp.headers['location']))
url = resp.headers['location']
else:
break
return fetchers.HTTPResponse(url, resp.status_code, resp.headers,
resp.content)
def test_redirect(self):
# Test redirect response - a final response comes from another URL.
with responses.RequestsMock() as rsps:
rsps.add(responses.GET,
'http://example.cz/redirect/',
status=302,
headers={'Location': 'http://example.cz/target/'})
rsps.add(responses.GET,
'http://example.cz/target/',
status=200,
body=b'BODY',
headers={'Content-Type': 'text/plain'})
response = self.fetcher.fetch('http://example.cz/redirect/')
expected = fetchers.HTTPResponse('http://example.cz/target/', 200,
{'Content-Type': 'text/plain'},
b'BODY')
assertResponse(expected, response)
def fetch(self, url, headers, body):
current_url = url
while True:
parsed = urllib.parse.urllib.parse(current_url)
path = parsed[2][1:]
try:
data = discoverdata.generateSample(path, self.base_url)
except KeyError:
return fetchers.HTTPResponse(
status=404, final_url=current_url, headers={}, body='')
response = mkResponse(data)
if response.status in [301, 302, 303, 307]:
current_url = response.headers['location']
else:
response.final_url = current_url
return response
def fetch(self, url, body=None, headers=None):
if headers is None:
headers = {}
headers.setdefault(
'User-Agent',
"%s Python-urlfetch" % (fetchers.USER_AGENT))
f = urlfetch.fetch(url,
method=(urlfetch.POST if body else urlfetch.GET),
headers=headers,
payload=body,
validate_certificate=True)
resp = fetchers.HTTPResponse()
resp.body = f.content
resp.final_url = f.final_url or url
resp.headers = f.headers
resp.status = f.status_code
return resp
def fetch(self, url, body=None, headers=None):
if not fetchers._allowedURL(url):
raise ValueError('Bad URL scheme: %r' % (url,))
if not headers:
headers = {}
if body:
method = urlfetch.POST
headers['Content-type'] = 'application/x-www-form-urlencoded'
else:
method = urlfetch.GET
count = 0
resp = urlfetch.fetch(url, body, method, headers=headers)
# follow redirects for a while
while resp.status_code in [301,302]:
count += 1
if count >= 3:
raise Exception('too many redirects')
if resp.headers.has_key('location'):
url = resp.headers['location']
elif resp.headers.has_key('Location'):
url = resp.headers['Location']
else:
raise Exception('Could not find location in headers: %r' % (resp.headers,))
resp = urlfetch.fetch(url, body, method, headers=headers)
# normalize headers
for key, val in resp.headers.items():
resp.headers[key.lower()] = val
return fetchers.HTTPResponse(url, resp.status_code, resp.headers,
resp.content)
def plain(path, code):
path = '/' + path
expected = fetchers.HTTPResponse(geturl(path), code, expected_headers,
path)
return (path, expected)
