def test_disallowed(self):
'''
Test that the _allowedURL function only lets through the right things.
'''
for url in ["file://localhost/thing.txt", "ftp://server/path",
"sftp://server/path", "ssh://server/path"]:
self.assertEqual(fetchers._allowedURL(url), False)
def test_disallowed(self):
'''
Test that the _allowedURL function only lets through the right things.
'''
for url in [
"file://localhost/thing.txt", "ftp://server/path",
"sftp://server/path", "ssh://server/path"
]:
self.assertEqual(fetchers._allowedURL(url), False)
def fetch(self, url, body=None, headers=None):
"""
This performs an HTTP POST or GET, following redirects along
the way. If a body is specified, then the request will be a
POST. Otherwise, it will be a GET.
@param headers: HTTP headers to include with the request
@type headers: {str:str}
@return: An object representing the server's HTTP response. If
there are network or protocol errors, an exception will be
raised. HTTP error responses, like 404 or 500, do not
cause exceptions.
@rtype: L{HTTPResponse}
@raise Exception: Different implementations will raise
different errors based on the underlying HTTP library.
"""
if not fetchers._allowedURL(url):
raise ValueError('Bad URL scheme: %r' % (url,))
if body:
method = urlfetch.POST
if headers is not None and 'Content-Type' not in headers:
headers['Content-Type'] = 'application/x-www-form-urlencoded'
else:
method = urlfetch.GET
if not headers:
headers = {}
# follow up to 10 redirects
for i in range(10):
resp = urlfetch.fetch(url, body, method, headers)
if resp.status_code in (301, 302):
logging.debug('Following %d redirect to %s' %
(resp.status_code, resp.headers['location']))
#url = resp.headers['location']
if resp.headers.has_key('location'):
url = resp.headers['location']
elif resp.headers.has_key('Location'):
url = resp.headers['Location']
else:
raise Exception('Could not find location in headers: %r' % (resp.headers,))
else:
break
return fetchers.HTTPResponse(url, resp.status_code, resp.headers,
resp.content)
def fetch(self, url, body=None, headers=None):
"""
This performs an HTTP POST or GET, following redirects along
the way. If a body is specified, then the request will be a
POST. Otherwise, it will be a GET.
@param headers: HTTP headers to include with the request
@type headers: {str:str}
@return: An object representing the server's HTTP response. If
there are network or protocol errors, an exception will be
raised. HTTP error responses, like 404 or 500, do not
cause exceptions.
@rtype: L{HTTPResponse}
@raise Exception: Different implementations will raise
different errors based on the underlying HTTP library.
"""
if not fetchers._allowedURL(url):
raise ValueError('Bad URL scheme: %r' % (url,))
if not headers:
headers = {}
if body:
method = urlfetch.POST
if 'Content-Type' not in headers:
headers['Content-Type'] = 'application/x-www-form-urlencoded'
else:
method = urlfetch.GET
if not headers:
headers = {}
# follow up to 10 redirects
for i in range(10):
resp = urlfetch.fetch(url, body, method, headers)
if resp.status_code in (301, 302):
logging.debug('Following %d redirect to %s' %
(resp.status_code, resp.headers['location']))
url = resp.headers['location']
else:
break
return fetchers.HTTPResponse(url, resp.status_code, resp.headers,
resp.content)
def fetch(self, url, body=None, headers=None):
if not fetchers._allowedURL(url):
raise ValueError('Bad URL scheme: %r' % (url,))
if not headers:
headers = {}
if body:
method = urlfetch.POST
headers['Content-type'] = 'application/x-www-form-urlencoded'
else:
method = urlfetch.GET
count = 0
resp = urlfetch.fetch(url, body, method, headers=headers)
# follow redirects for a while
while resp.status_code in [301,302]:
count += 1
if count >= 3:
raise Exception('too many redirects')
if resp.headers.has_key('location'):
url = resp.headers['location']
elif resp.headers.has_key('Location'):
url = resp.headers['Location']
else:
raise Exception('Could not find location in headers: %r' % (resp.headers,))
resp = urlfetch.fetch(url, body, method, headers=headers)
# normalize headers
for key, val in resp.headers.items():
resp.headers[key.lower()] = val
return fetchers.HTTPResponse(url, resp.status_code, resp.headers,
resp.content)
def fetch(self, url, body=None, headers=None):
if not fetchers._allowedURL(url):
raise ValueError('Bad URL scheme: %r' % (url,))
if not headers:
headers = {}
if body:
method = urlfetch.POST
headers['Content-type'] = 'application/x-www-form-urlencoded'
else:
method = urlfetch.GET
count = 0
resp = urlfetch.fetch(url, body, method, headers=headers)
# follow redirects for a while
while resp.status_code in [301,302]:
count += 1
if count >= 3:
raise Exception('too many redirects')
if resp.headers.has_key('location'):
url = resp.headers['location']
elif resp.headers.has_key('Location'):
url = resp.headers['Location']
else:
raise Exception('Could not find location in headers: %r' % (resp.headers,))
resp = urlfetch.fetch(url, body, method, headers=headers)
# normalize headers
for key, val in resp.headers.items():
resp.headers[key.lower()] = val
return fetchers.HTTPResponse(url, resp.status_code, resp.headers,
resp.content)