def fetch(self, uri, headers=None, body=None): self.count += 1 if self.count == 1: headers = { 'X-XRDS-Location'.lower(): 'http://unittest/404', } return fetchers.HTTPResponse(uri, 200, headers, '') else: return fetchers.HTTPResponse(uri, 404)
def test_fetcher(fetcher, exc, server): def geturl(path): host, port = server.server_address return 'http://%s:%s%s' % (host, port, path) expected_headers = {'content-type': 'text/plain'} def plain(path, code): path = '/' + path expected = fetchers.HTTPResponse(geturl(path), code, expected_headers, path) return (path, expected) expect_success = fetchers.HTTPResponse(geturl('/success'), 200, expected_headers, '/success') cases = [ ('/success', expect_success), ('/301redirect', expect_success), ('/302redirect', expect_success), ('/303redirect', expect_success), ('/307redirect', expect_success), plain('notfound', 404), plain('badreq', 400), plain('forbidden', 403), plain('error', 500), plain('server_error', 503), ] for path, expected in cases: fetch_url = geturl(path) try: actual = fetcher.fetch(fetch_url) except (SystemExit, KeyboardInterrupt): pass except Exception as e: raise AssertionError((fetcher, fetch_url, e)) raise else: failUnlessResponseExpected(expected, actual, extra=locals()) for err_url in [ geturl('/closed'), 'http://invalid.janrain.com/', 'not:a/url', 'ftp://janrain.com/pub/' ]: try: result = fetcher.fetch(err_url) except (KeyboardInterrupt, SystemExit): raise except fetchers.HTTPError: # This is raised by the Curl fetcher for bad cases # detected by the fetchers module, but it's a subclass of # HTTPFetchingError, so we have to catch it explicitly. assert exc except fetchers.HTTPFetchingError: assert not exc, (fetcher, exc, server) except: assert exc else: assert False, 'An exception was expected for %r (%r)' % (fetcher, result)
def test_error(self): # Test error responses - returned as obtained self.add_response('http://example.cz/error/', 500, {'Content-Type': 'text/plain'}, 'BODY') response = self.fetcher.fetch('http://example.cz/error/') expected = fetchers.HTTPResponse('http://example.cz/error/', 500, {'Content-Type': 'text/plain'}, 'BODY') assertResponse(expected, response)
def test_redirect(self): # Test redirect response - a final response comes from another URL. self.add_response('http://example.cz/success/', 200, {'Content-Type': 'text/plain'}, 'BODY') response = self.fetcher.fetch('http://example.cz/redirect/') expected = fetchers.HTTPResponse('http://example.cz/success/', 200, {'Content-Type': 'text/plain'}, 'BODY') assertResponse(expected, response)
def test_success(self): # Test success response self.add_response('http://example.cz/success/', 200, {'Content-Type': 'text/plain'}, 'BODY') response = self.fetcher.fetch('http://example.cz/success/') expected = fetchers.HTTPResponse('http://example.cz/success/', 200, {'Content-Type': 'text/plain'}, 'BODY') assertResponse(expected, response)
def test_fetcher(fetcher, exc, server): def geturl(path): return 'http://%s:%s%s' % (socket.getfqdn( server.server_name), server.socket.getsockname()[1], path) expected_headers = {'content-type': 'text/plain'} def plain(path, code): path = '/' + path expected = fetchers.HTTPResponse(geturl(path), code, expected_headers, path.encode('utf-8')) return (path, expected) expect_success = fetchers.HTTPResponse(geturl('/success'), 200, expected_headers, b'/success') cases = [ ('/success', expect_success), ('/301redirect', expect_success), ('/302redirect', expect_success), ('/303redirect', expect_success), ('/307redirect', expect_success), plain('notfound', 404), plain('badreq', 400), plain('forbidden', 403), plain('error', 500), plain('server_error', 503), ] for path, expected in cases: fetch_url = geturl(path) try: actual = fetcher.fetch(fetch_url) except Exception: print(fetcher, fetch_url) raise else: assertResponse(expected, actual) for err_url in [ geturl('/closed'), 'http://invalid.janrain.com/', 'not:a/url', 'ftp://janrain.com/pub/' ]: try: result = fetcher.fetch(err_url) except fetchers.HTTPError: # This is raised by the Curl fetcher for bad cases # detected by the fetchers module, but it's a subclass of # HTTPFetchingError, so we have to catch it explicitly. assert exc except fetchers.HTTPFetchingError: assert not exc, (fetcher, exc, server) except Exception: assert exc else: assert False, 'An exception was expected for %r (%r)' % (fetcher, result)
def test_connection_error(self): # Test connection error self.http_mock.side_effect = urllib2.HTTPError( 'http://example.cz/error/', 500, 'Error message', {'Content-Type': 'text/plain'}, StringIO('BODY')) response = self.fetcher.fetch('http://example.cz/error/') expected = fetchers.HTTPResponse('http://example.cz/error/', 500, {'Content-Type': 'text/plain'}, 'BODY') assertResponse(expected, response)
def mkResponse(data): status_mo = status_header_re.match(data) headers_str, body = data.split('\n\n', 1) headers = {} for line in headers_str.split('\n'): k, v = line.split(':', 1) k = k.strip().lower() v = v.strip() headers[k] = v status = int(status_mo.group(1)) return fetchers.HTTPResponse(status=status, headers=headers, body=body)
def test_error(self): # Test error responses - returned as obtained with responses.RequestsMock() as rsps: rsps.add(responses.GET, 'http://example.cz/error/', status=500, body=b'BODY', headers={'Content-Type': 'text/plain'}) response = self.fetcher.fetch('http://example.cz/error/') expected = fetchers.HTTPResponse('http://example.cz/error/', 500, {'Content-Type': 'text/plain'}, b'BODY') assertResponse(expected, response)
def test_get(self): # Test GET response with responses.RequestsMock() as rsps: rsps.add(responses.GET, 'http://example.cz/', status=200, body=b'BODY', headers={'Content-Type': 'text/plain'}) response = self.fetcher.fetch('http://example.cz/') expected = fetchers.HTTPResponse('http://example.cz/', 200, {'Content-Type': 'text/plain'}, b'BODY') assertResponse(expected, response)
def fetch(self, url, body=None, headers=None): """ This performs an HTTP POST or GET, following redirects along the way. If a body is specified, then the request will be a POST. Otherwise, it will be a GET. @param headers: HTTP headers to include with the request @type headers: {str:str} @return: An object representing the server's HTTP response. If there are network or protocol errors, an exception will be raised. HTTP error responses, like 404 or 500, do not cause exceptions. @rtype: L{HTTPResponse} @raise Exception: Different implementations will raise different errors based on the underlying HTTP library. """ if not fetchers._allowedURL(url): raise ValueError('Bad URL scheme: %r' % (url,)) if not headers: headers = {} if body: method = urlfetch.POST if 'Content-Type' not in headers: headers['Content-Type'] = 'application/x-www-form-urlencoded' else: method = urlfetch.GET if not headers: headers = {} # follow up to 10 redirects for i in range(10): resp = urlfetch.fetch(url, body, method, headers) if resp.status_code in (301, 302): logging.debug('Following %d redirect to %s' % (resp.status_code, resp.headers['location'])) url = resp.headers['location'] else: break return fetchers.HTTPResponse(url, resp.status_code, resp.headers, resp.content)
def test_redirect(self): # Test redirect response - a final response comes from another URL. with responses.RequestsMock() as rsps: rsps.add(responses.GET, 'http://example.cz/redirect/', status=302, headers={'Location': 'http://example.cz/target/'}) rsps.add(responses.GET, 'http://example.cz/target/', status=200, body=b'BODY', headers={'Content-Type': 'text/plain'}) response = self.fetcher.fetch('http://example.cz/redirect/') expected = fetchers.HTTPResponse('http://example.cz/target/', 200, {'Content-Type': 'text/plain'}, b'BODY') assertResponse(expected, response)
def fetch(self, url, headers, body): current_url = url while True: parsed = urllib.parse.urllib.parse(current_url) path = parsed[2][1:] try: data = discoverdata.generateSample(path, self.base_url) except KeyError: return fetchers.HTTPResponse( status=404, final_url=current_url, headers={}, body='') response = mkResponse(data) if response.status in [301, 302, 303, 307]: current_url = response.headers['location'] else: response.final_url = current_url return response
def fetch(self, url, body=None, headers=None): if headers is None: headers = {} headers.setdefault( 'User-Agent', "%s Python-urlfetch" % (fetchers.USER_AGENT)) f = urlfetch.fetch(url, method=(urlfetch.POST if body else urlfetch.GET), headers=headers, payload=body, validate_certificate=True) resp = fetchers.HTTPResponse() resp.body = f.content resp.final_url = f.final_url or url resp.headers = f.headers resp.status = f.status_code return resp
def fetch(self, url, body=None, headers=None): if not fetchers._allowedURL(url): raise ValueError('Bad URL scheme: %r' % (url,)) if not headers: headers = {} if body: method = urlfetch.POST headers['Content-type'] = 'application/x-www-form-urlencoded' else: method = urlfetch.GET count = 0 resp = urlfetch.fetch(url, body, method, headers=headers) # follow redirects for a while while resp.status_code in [301,302]: count += 1 if count >= 3: raise Exception('too many redirects') if resp.headers.has_key('location'): url = resp.headers['location'] elif resp.headers.has_key('Location'): url = resp.headers['Location'] else: raise Exception('Could not find location in headers: %r' % (resp.headers,)) resp = urlfetch.fetch(url, body, method, headers=headers) # normalize headers for key, val in resp.headers.items(): resp.headers[key.lower()] = val return fetchers.HTTPResponse(url, resp.status_code, resp.headers, resp.content)
def plain(path, code): path = '/' + path expected = fetchers.HTTPResponse(geturl(path), code, expected_headers, path) return (path, expected)