Ejemplo n.º 1
0
 def test_disallowed(self):
     '''
     Test that the _allowedURL function only lets through the right things.
     '''
     for url in ["file://localhost/thing.txt", "ftp://server/path",
                 "sftp://server/path", "ssh://server/path"]:
         self.assertEqual(fetchers._allowedURL(url), False)
Ejemplo n.º 2
0
 def test_disallowed(self):
     '''
     Test that the _allowedURL function only lets through the right things.
     '''
     for url in [
             "file://localhost/thing.txt", "ftp://server/path",
             "sftp://server/path", "ssh://server/path"
     ]:
         self.assertEqual(fetchers._allowedURL(url), False)
Ejemplo n.º 3
0
  def fetch(self, url, body=None, headers=None):
    """
    This performs an HTTP POST or GET, following redirects along
    the way. If a body is specified, then the request will be a
    POST. Otherwise, it will be a GET.

    @param headers: HTTP headers to include with the request
    @type headers: {str:str}

    @return: An object representing the server's HTTP response. If
      there are network or protocol errors, an exception will be
      raised. HTTP error responses, like 404 or 500, do not
      cause exceptions.

    @rtype: L{HTTPResponse}

    @raise Exception: Different implementations will raise
      different errors based on the underlying HTTP library.
    """
    if not fetchers._allowedURL(url):
      raise ValueError('Bad URL scheme: %r' % (url,))

    if body:
      method = urlfetch.POST
      if headers is not None and 'Content-Type' not in headers:
        headers['Content-Type'] = 'application/x-www-form-urlencoded'
    else:
      method = urlfetch.GET

    if not headers:
      headers = {}

    # follow up to 10 redirects
    for i in range(10):
      resp = urlfetch.fetch(url, body, method, headers)
      if resp.status_code in (301, 302):
        logging.debug('Following %d redirect to %s' %
                      (resp.status_code, resp.headers['location']))
        #url = resp.headers['location']
        if resp.headers.has_key('location'):
            url = resp.headers['location']
        elif resp.headers.has_key('Location'):
            url = resp.headers['Location']
        else:
            raise Exception('Could not find location in headers: %r' % (resp.headers,))

      else:
        break

    return fetchers.HTTPResponse(url, resp.status_code, resp.headers,
                                 resp.content)
  def fetch(self, url, body=None, headers=None):
    """
    This performs an HTTP POST or GET, following redirects along
    the way. If a body is specified, then the request will be a
    POST. Otherwise, it will be a GET.

    @param headers: HTTP headers to include with the request
    @type headers: {str:str}

    @return: An object representing the server's HTTP response. If
      there are network or protocol errors, an exception will be
      raised. HTTP error responses, like 404 or 500, do not
      cause exceptions.

    @rtype: L{HTTPResponse}

    @raise Exception: Different implementations will raise
      different errors based on the underlying HTTP library.
    """
    if not fetchers._allowedURL(url):
      raise ValueError('Bad URL scheme: %r' % (url,))

    if not headers:
      headers = {}

    if body:
      method = urlfetch.POST
      if 'Content-Type' not in headers:
        headers['Content-Type'] = 'application/x-www-form-urlencoded'
    else:
      method = urlfetch.GET

    if not headers:
      headers = {}

    # follow up to 10 redirects
    for i in range(10):
      resp = urlfetch.fetch(url, body, method, headers)
      if resp.status_code in (301, 302):
        logging.debug('Following %d redirect to %s' %
                      (resp.status_code, resp.headers['location']))
        url = resp.headers['location']
      else:
        break

    return fetchers.HTTPResponse(url, resp.status_code, resp.headers,
                                 resp.content)
Ejemplo n.º 5
0
  def fetch(self, url, body=None, headers=None):
    if not fetchers._allowedURL(url):
      raise ValueError('Bad URL scheme: %r' % (url,))
    
    if not headers:
      headers = {}

    if body:
      method = urlfetch.POST
      headers['Content-type'] = 'application/x-www-form-urlencoded' 
    else:
      method = urlfetch.GET

    count = 0
    resp = urlfetch.fetch(url, body, method, headers=headers)

    # follow redirects for a while
    while resp.status_code in [301,302]:
      count += 1
      if count >= 3:
        raise Exception('too many redirects')

      if resp.headers.has_key('location'):
        url = resp.headers['location']
      elif resp.headers.has_key('Location'):
        url = resp.headers['Location']
      else:
        raise Exception('Could not find location in headers: %r' % (resp.headers,))
      
      resp = urlfetch.fetch(url, body, method, headers=headers)

    # normalize headers
    for key, val in resp.headers.items():
      resp.headers[key.lower()] = val

    return fetchers.HTTPResponse(url, resp.status_code, resp.headers,
                                 resp.content)
Ejemplo n.º 6
0
  def fetch(self, url, body=None, headers=None):
    if not fetchers._allowedURL(url):
      raise ValueError('Bad URL scheme: %r' % (url,))
    
    if not headers:
      headers = {}

    if body:
      method = urlfetch.POST
      headers['Content-type'] = 'application/x-www-form-urlencoded' 
    else:
      method = urlfetch.GET

    count = 0
    resp = urlfetch.fetch(url, body, method, headers=headers)

    # follow redirects for a while
    while resp.status_code in [301,302]:
      count += 1
      if count >= 3:
        raise Exception('too many redirects')

      if resp.headers.has_key('location'):
        url = resp.headers['location']
      elif resp.headers.has_key('Location'):
        url = resp.headers['Location']
      else:
        raise Exception('Could not find location in headers: %r' % (resp.headers,))
      
      resp = urlfetch.fetch(url, body, method, headers=headers)

    # normalize headers
    for key, val in resp.headers.items():
      resp.headers[key.lower()] = val

    return fetchers.HTTPResponse(url, resp.status_code, resp.headers,
                                 resp.content)