def test_disallowed(self): ''' Test that the _allowedURL function only lets through the right things. ''' for url in ["file://localhost/thing.txt", "ftp://server/path", "sftp://server/path", "ssh://server/path"]: self.assertEqual(fetchers._allowedURL(url), False)
def test_disallowed(self): ''' Test that the _allowedURL function only lets through the right things. ''' for url in [ "file://localhost/thing.txt", "ftp://server/path", "sftp://server/path", "ssh://server/path" ]: self.assertEqual(fetchers._allowedURL(url), False)
def fetch(self, url, body=None, headers=None): """ This performs an HTTP POST or GET, following redirects along the way. If a body is specified, then the request will be a POST. Otherwise, it will be a GET. @param headers: HTTP headers to include with the request @type headers: {str:str} @return: An object representing the server's HTTP response. If there are network or protocol errors, an exception will be raised. HTTP error responses, like 404 or 500, do not cause exceptions. @rtype: L{HTTPResponse} @raise Exception: Different implementations will raise different errors based on the underlying HTTP library. """ if not fetchers._allowedURL(url): raise ValueError('Bad URL scheme: %r' % (url,)) if body: method = urlfetch.POST if headers is not None and 'Content-Type' not in headers: headers['Content-Type'] = 'application/x-www-form-urlencoded' else: method = urlfetch.GET if not headers: headers = {} # follow up to 10 redirects for i in range(10): resp = urlfetch.fetch(url, body, method, headers) if resp.status_code in (301, 302): logging.debug('Following %d redirect to %s' % (resp.status_code, resp.headers['location'])) #url = resp.headers['location'] if resp.headers.has_key('location'): url = resp.headers['location'] elif resp.headers.has_key('Location'): url = resp.headers['Location'] else: raise Exception('Could not find location in headers: %r' % (resp.headers,)) else: break return fetchers.HTTPResponse(url, resp.status_code, resp.headers, resp.content)
def fetch(self, url, body=None, headers=None): """ This performs an HTTP POST or GET, following redirects along the way. If a body is specified, then the request will be a POST. Otherwise, it will be a GET. @param headers: HTTP headers to include with the request @type headers: {str:str} @return: An object representing the server's HTTP response. If there are network or protocol errors, an exception will be raised. HTTP error responses, like 404 or 500, do not cause exceptions. @rtype: L{HTTPResponse} @raise Exception: Different implementations will raise different errors based on the underlying HTTP library. """ if not fetchers._allowedURL(url): raise ValueError('Bad URL scheme: %r' % (url,)) if not headers: headers = {} if body: method = urlfetch.POST if 'Content-Type' not in headers: headers['Content-Type'] = 'application/x-www-form-urlencoded' else: method = urlfetch.GET if not headers: headers = {} # follow up to 10 redirects for i in range(10): resp = urlfetch.fetch(url, body, method, headers) if resp.status_code in (301, 302): logging.debug('Following %d redirect to %s' % (resp.status_code, resp.headers['location'])) url = resp.headers['location'] else: break return fetchers.HTTPResponse(url, resp.status_code, resp.headers, resp.content)
def fetch(self, url, body=None, headers=None): if not fetchers._allowedURL(url): raise ValueError('Bad URL scheme: %r' % (url,)) if not headers: headers = {} if body: method = urlfetch.POST headers['Content-type'] = 'application/x-www-form-urlencoded' else: method = urlfetch.GET count = 0 resp = urlfetch.fetch(url, body, method, headers=headers) # follow redirects for a while while resp.status_code in [301,302]: count += 1 if count >= 3: raise Exception('too many redirects') if resp.headers.has_key('location'): url = resp.headers['location'] elif resp.headers.has_key('Location'): url = resp.headers['Location'] else: raise Exception('Could not find location in headers: %r' % (resp.headers,)) resp = urlfetch.fetch(url, body, method, headers=headers) # normalize headers for key, val in resp.headers.items(): resp.headers[key.lower()] = val return fetchers.HTTPResponse(url, resp.status_code, resp.headers, resp.content)