def request(self, *args, **kwargs): if failures: try: raise failures.pop(0) except: raise http_error.RequestError(util.SavedException()) else: return MockResponse.OK
def _handleProxyErrors(self, errcode): """Translate proxy error codes into exceptions.""" if errcode == 503: # Service unavailable, make it a socket error e = socket.error(111, "Service unavailable") elif errcode == 502: # Bad gateway (server responded with some broken answer) e = socket.error(111, "Bad Gateway (error reported by proxy)") else: return # Proxy errors are treated as request errors, which are retriable. saved = util.SavedException(e) raise http_error.RequestError(saved)
def requestOnce(self, conn, req): if self.proxy and self.proxy.userpass[0] and not self.doTunnel: req.headers['Proxy-Authorization'] = ( 'Basic ' + base64.b64encode(":".join(self.proxy.userpass))) try: req.sendRequest(conn, isProxied=(self.proxy is not None)) except (KeyboardInterrupt, SystemExit): raise except: wrapped = util.SavedException() raise http_error.RequestError(wrapped) # Wait for a response. poller = select.poll() poller.register(conn.sock.fileno(), select.POLLIN) lastTimeout = time.time() while True: if req.abortCheck and req.abortCheck(): raise http_error.AbortError() # Wait 5 seconds for a response. try: active = poller.poll(5000) except select.error, err: if err.args[0] == errno.EINTR: # Interrupted system call -- we caught a signal but it was # handled safely. continue raise if active: break # Still no response from the server. Send blank lines to keep the # connection alive, in case the server is behind a load balancer or # firewall with short connection timeouts. now = time.time() if now - lastTimeout >= 15: conn.send('\r\n') lastTimeout = now
class Connection(object): """Connection to a single endpoint, possibly encrypted and/or proxied and/or tunneled. May be kept alive betwen requests and reopened if a kept-alive connection fails on subsequent use. Will not attempt to retry on other network errors, nor will it interpret HTTP responses. """ userAgent = "conary-http-client/%s" % constants.version connectTimeout = 30 def __init__(self, endpoint, proxy=None, caCerts=None, commonName=None): """ @param endpoint: Destination URL (host, port, optional SSL, optional authorization) @param proxy: Optional proxy URL (host, port, optional SSL, optional authorization) @param caCerts: Optional list of CA certificate paths to check servers against. @param commonName: Optional hostname to use for checking server certificates. """ # endpoint and proxy must be URL objects, not names. self.endpoint = endpoint self.proxy = proxy self.caCerts = caCerts if proxy: self.local = proxy else: self.local = endpoint if commonName is None: commonName = str(self.endpoint.hostport.host) self.commonName = commonName self.doSSL = endpoint.scheme == 'https' self.doTunnel = bool(proxy) and self.doSSL # Cached HTTPConnection object self.cached = None def close(self): if self.cached: self.cached.close() self.cached = None def request(self, req): if self.cached: # Try once to use the cached connection; if it fails to send the # request then discard and try again. try: return self.requestOnce(self.cached, req) except http_error.RequestError, err: err.wrapped.clear() self.cached.close() self.cached = None # If a problem occurs before or during the sending of the request, then # throw a wrapper exception so that the caller knows it is safe to # retry. Once the request is sent retries must be done more carefully # as side effects may have occurred. try: conn = self.openConnection() except (KeyboardInterrupt, SystemExit): raise except: wrapped = util.SavedException() raise http_error.RequestError(wrapped) # Note that requestOnce may also throw RequestError, see above. ret = self.requestOnce(conn, req) if not ret.will_close: self.cached = conn return ret
def _doRequest(self, req, forceProxy): resetResolv = False lastError = response = None timer = timeutil.BackoffTimer() totalAttempts = 0 # Make at least 'connectAttempts' connection attempts, stopping after # both passing the connectAttempts limit *and* hitting the end of the # iterator. while True: if totalAttempts >= self.connectAttempts: break # Reset the failed proxy list each time around so we don't end up # blacklisting everything if a second pass succeeds. failedProxies = set() if forceProxy is False: connIterator = self.proxyMap.getProxyIter( req.url, protocolFilter=self.proxyFilter) else: connIterator = [forceProxy] for proxySpec in connIterator: totalAttempts += 1 if proxySpec == proxy_map.DirectConnection: proxySpec = None elif not forceProxy and self._shouldBypass(req.url, proxySpec): proxySpec = None if lastError: if proxySpec == self.lastProxy: log.debug("Failed to open URL %s; trying again: %s", req.url, lastError.format()) else: log.info( "Failed to open URL %s; trying the next " "proxy: %s", req.url, lastError.format()) try: req.reset() except: log.exception("Failed to rewind request body file, " "unable to retry request:") break # If a proxy was used, save it here self.lastProxy = proxySpec try: response = self._requestOnce(req, proxySpec) break except http_error.RequestError, err: # Retry if an error occurred while sending the request. lastError = err.wrapped err = lastError.value if lastError.check(socket.error): self._processSocketError(err) lastError.replace(err) except httplib.BadStatusLine: # closed connection without sending a response. lastError = util.SavedException() except socket.error, err: # Fatal error, but attach proxy information to it. self._processSocketError(err) util.rethrow(err, False)