def _useProxyFor(url): """Return True if a proxy should be used for given url, otherwise False. This function uses server.satellite.no_proxy variable to check for hosts or domains which should not be connected via a proxy. server.satellite.no_proxy is a comma seperated list. Either an exact match, or the previous character is a '.', so host is within the same domain. A leading '.' in the pattern is ignored. See also 'man curl' """ u = urlparse.urlsplit(url) # pylint can't see inside the SplitResult class # pylint: disable=E1103 if u.scheme == 'file': return False hostname = u.hostname.lower() if hostname in ["localhost", "127.0.0.1", "::1"]: return False comp = CFG.getComponent() if not CFG.has_key("no_proxy"): initCFG("server.satellite") if not CFG.has_key('no_proxy'): initCFG(comp) return True noproxy = CFG.no_proxy initCFG(comp) if not noproxy: return True if not isinstance(noproxy, list): if noproxy == '*': # just an asterisk disables all. return False noproxy = [noproxy] # No proxy: Either an exact match, or the previous character # is a '.', so host is within the same domain. # A leading '.' in the pattern is ignored. Some implementations # need '.foo.ba' to prevent 'foo.ba' from matching 'xfoo.ba'. for domain in noproxy: domain = domain.lower() if domain[0] == '.': domain = domain[1:] if hostname.endswith(domain) and \ (len(hostname) == len(domain) or hostname[len(hostname) - len(domain) - 1] == '.'): return False return True
def _create_connection(self): """ Returns a Connection object """ scheme, host, port, _uri = self._parse_url(self.rhnParent) # Build the list of params params = { 'host': host, 'port': port, } if CFG.has_key('timeout'): params['timeout'] = CFG.TIMEOUT if self.httpProxy: params['proxy'] = self.httpProxy params['username'] = self.httpProxyUsername params['password'] = self.httpProxyPassword if scheme == 'https' and self.caChain: params['trusted_certs'] = [self.caChain, ] # Now select the right class if self.httpProxy: if scheme == 'https': conn_class = connections.HTTPSProxyConnection else: conn_class = connections.HTTPProxyConnection else: if scheme == 'https': conn_class = connections.HTTPSConnection else: conn_class = connections.HTTPConnection log_debug(5, "Using connection class", conn_class, 'Params:', params) return conn_class(**params)
def _create_connection(self): """ Returns a Connection object """ scheme, host, port, _uri = self._parse_url(self.rhnParent) # Build the list of params params = {"host": host, "port": port} if CFG.has_key("timeout"): params["timeout"] = CFG.TIMEOUT if self.httpProxy: params["proxy"] = self.httpProxy params["username"] = self.httpProxyUsername params["password"] = self.httpProxyPassword if scheme == "https" and self.caChain: params["trusted_certs"] = [self.caChain] # Now select the right class if self.httpProxy: if scheme == "https": conn_class = connections.HTTPSProxyConnection else: conn_class = connections.HTTPProxyConnection else: if scheme == "https": conn_class = connections.HTTPSConnection else: conn_class = connections.HTTPConnection log_debug(5, "Using connection class", conn_class, "Params:", params) return conn_class(**params)
def accessible(url): """Try if url is accessible :arg url: the url which is tried to access Returns True if url is accessible, otherwise False. """ timeout = 120 if CFG.is_initialized() and CFG.has_key('TIMEOUT'): timeout = CFG.TIMEOUT curl = pycurl.Curl() curl.setopt(pycurl.CONNECTTIMEOUT, timeout) curl.setopt(pycurl.URL, url) curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug) curl.setopt(pycurl.VERBOSE, True) proxy_url, proxy_user, proxy_pass = get_proxy(url) if proxy_url: curl.setopt(pycurl.PROXY, proxy_url) log_debug(2, "Connect to %s" % url) # We implement our own redirection-following, because pycurl # 7.19 doesn't POST after it gets redirected. Ideally we'd be # using pycurl.POSTREDIR here, but that's in 7.21. curl.setopt(pycurl.FOLLOWLOCATION, False) curl.setopt(pycurl.NOBODY, True) try_counter = 5 while try_counter: try_counter -= 1 try: curl.perform() except pycurl.error as e: if e[0] == 56: # Proxy requires authentication log_debug(2, e[1]) if not (proxy_user and proxy_pass): raise TransferException("Proxy requires authentication, " "but reading credentials from " "%s failed." % YAST_PROXY) curl.setopt(pycurl.PROXYUSERPWD, "%s:%s" % (proxy_user, proxy_pass)) else: break status = curl.getinfo(pycurl.HTTP_CODE) # OK or file if status == 200 or (URL(url).scheme == "file" and status == 0): return True elif status in (301, 302): # redirects url = curl.getinfo(pycurl.REDIRECT_URL) log_debug(2, "Got redirect to %s" % url) curl.setopt(pycurl.URL, url) elif status >= 400: break return False
def accessible(url): """Try if url is accessible :arg url: the url which is tried to access Returns True if url is accessible, otherwise False. """ timeout = 120 if CFG.is_initialized() and CFG.has_key('TIMEOUT'): timeout = CFG.TIMEOUT curl = pycurl.Curl() curl.setopt(pycurl.CONNECTTIMEOUT, timeout) curl.setopt(pycurl.URL, url) curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug) curl.setopt(pycurl.VERBOSE, True) proxy_url, proxy_user, proxy_pass = get_proxy(url) if proxy_url: curl.setopt(pycurl.PROXY, proxy_url) log_debug(2, "Connect to %s" % url) # We implement our own redirection-following, because pycurl # 7.19 doesn't POST after it gets redirected. Ideally we'd be # using pycurl.POSTREDIR here, but that's in 7.21. curl.setopt(pycurl.FOLLOWLOCATION, False) curl.setopt(pycurl.NOBODY, True) try_counter = 5 while try_counter: try_counter -= 1 try: curl.perform() except pycurl.error, e: if e[0] == 56: # Proxy requires authentication log_debug(2, e[1]) if not (proxy_user and proxy_pass): raise TransferException("Proxy requires authentication, " "but reading credentials from " "%s failed." % YAST_PROXY) curl.setopt(pycurl.PROXYUSERPWD, "%s:%s" % (proxy_user, proxy_pass)) else: break status = curl.getinfo(pycurl.HTTP_CODE) # OK or file if status == 200 or (URL(url).scheme == "file" and status == 0): return True elif status in (301, 302): # redirects url = curl.getinfo(pycurl.REDIRECT_URL) log_debug(2, "Got redirect to %s" % url) curl.setopt(pycurl.URL, url) elif status >= 400: break
def _createConnection(host, port, scheme): params = {'host': host, 'port': port} if CFG.has_key('timeout'): params['timeout'] = CFG.TIMEOUT if scheme == SCHEME_HTTPS: conn_class = connections.HTTPSConnection else: conn_class = connections.HTTPConnection return conn_class(**params)
def _createConnection(host, port, scheme): params = {"host": host, "port": port} if CFG.has_key("timeout"): params["timeout"] = CFG.TIMEOUT if scheme == SCHEME_HTTPS: conn_class = connections.HTTPSConnection else: conn_class = connections.HTTPConnection return conn_class(**params)
def _get_proxy_from_rhn_conf(): """Return a tuple of (url, user, pass) proxy information from rhn config Returns None instead of a tuple if there was no proxy url. user, pass can be None. """ comp = CFG.getComponent() if not CFG.has_key("http_proxy"): initCFG("server.satellite") result = None if CFG.http_proxy: # CFG.http_proxy format is <hostname>[:<port>] in 1.7 url = 'http://%s' % CFG.http_proxy result = (url, CFG.http_proxy_username, CFG.http_proxy_password) initCFG(comp) log_debug(2, "Could not read proxy URL from rhn config.") return result
def send(url, sendData=None): """Connect to url and return the result as stringIO :arg url: the url where the request will be sent :kwarg sendData: do a post-request when "sendData" is given. Returns the result as stringIO object. """ connect_retries = 10 try_counter = connect_retries timeout = 120 if CFG.is_initialized() and CFG.has_key('TIMEOUT'): timeout = CFG.TIMEOUT curl = pycurl.Curl() curl.setopt(pycurl.CONNECTTIMEOUT, timeout) curl.setopt(pycurl.URL, url) curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug) curl.setopt(pycurl.VERBOSE, True) proxy_url, proxy_user, proxy_pass = get_proxy(url) if proxy_url: curl.setopt(pycurl.PROXY, proxy_url) log_debug(2, "Connect to %s" % url) if sendData is not None: curl.setopt(pycurl.POSTFIELDS, sendData) if (CFG.is_initialized() and CFG.has_key('DISABLE_EXPECT') and CFG.DISABLE_EXPECT): # disable Expect header curl.setopt(pycurl.HTTPHEADER, ['Expect:']) # We implement our own redirection-following, because pycurl # 7.19 doesn't POST after it gets redirected. Ideally we'd be # using pycurl.POSTREDIR here, but that's in 7.21. curl.setopt(pycurl.FOLLOWLOCATION, False) response = StringIO() curl.setopt(pycurl.WRITEFUNCTION, response.write) try_counter = connect_retries while try_counter: try_counter -= 1 try: curl.perform() except pycurl.error, e: if e[0] == 56: # Proxy requires authentication log_debug(2, e[1]) if not (proxy_user and proxy_pass): raise TransferException("Proxy requires authentication, " "but reading credentials from " "%s failed." % YAST_PROXY) curl.setopt(pycurl.PROXYUSERPWD, "%s:%s" % (proxy_user, proxy_pass)) elif e[0] == 60: log_error("Peer certificate could not be authenticated " "with known CA certificates.") raise TransferException("Peer certificate could not be " "authenticated with known CA " "certificates.") else: log_error(e[1]) raise status = curl.getinfo(pycurl.HTTP_CODE) if status == 200 or (URL(url).scheme == "file" and status == 0): # OK or file break elif status in (301, 302): # redirects url = curl.getinfo(pycurl.REDIRECT_URL) log_debug(2, "Got redirect to %s" % url) curl.setopt(pycurl.URL, url)
def __redirectToNextLocationNoRetry(self, loopProtection=False): """ This function will perform a redirection to the next location, as specified in the last response's "Location" header. This function will return an actual HTTP response status code. If successful, it will return apache.HTTP_OK, not apache.OK. If unsuccessful, this function will simply return; no retries will be performed. The following error codes can be returned: HTTP_OK,HTTP_PARTIAL_CONTENT - Redirect successful. HTTP_MOVED_TEMPORARILY - Redirect was redirected again by 3rd party. HTTP_MOVED_PERMANENTLY - Redirect was redirected again by 3rd party. HTTP_INTERNAL_SERVER_ERROR - Error extracting redirect information HTTP_SERVICE_UNAVAILABLE - Could not connect to 3rd party server, connection was reset, or a read error occurred during communication. HTTP_* - Any other HTTP status code may also be returned. Upon successful completion of this function, a new responseContext will be created and pushed onto the stack. """ # Obtain the redirect location first before we replace the current # response context. It's contained in the Location header of the # previous response. redirectLocation = self._get_header(rhnConstants.HEADER_LOCATION) # We are about to redirect to a new location so now we'll push a new # response context before we return any errors. self.responseContext.add() # There should always be a redirect URL passed back to us. If not, # there's an error. if not redirectLocation: log_error(" No redirect location specified!") Traceback(mail=0) return apache.HTTP_INTERNAL_SERVER_ERROR # The _get_header function returns the value as a list. There should # always be exactly one location specified. redirectLocation = redirectLocation[0] log_debug(1, " Redirecting to: ", redirectLocation) # Tear apart the redirect URL. We need the scheme, the host, the # port (if not the default), and the URI. _scheme, host, port, uri, query = self._parse_url(redirectLocation) # Add back the query string if query: uri += '?' + query # Now create a new connection. We'll use SSL if configured to do # so. params = { 'host': host, 'port': port, } if CFG.has_key('timeout'): params['timeout'] = CFG.TIMEOUT if CFG.USE_SSL: log_debug(1, " Redirecting with SSL. Cert= ", self.caChain) params['trusted_certs'] = [self.caChain] connection = connections.HTTPSConnection(**params) else: log_debug(1, " Redirecting withOUT SSL.") connection = connections.HTTPConnection(**params) # Put the connection into the current response context. self.responseContext.setConnection(connection) # Now open the connection to the 3rd party server. log_debug(4, "Attempting to connect to 3rd party server...") try: connection.connect() except socket.error as e: log_error("Error opening redirect connection", redirectLocation, e) Traceback(mail=0) return apache.HTTP_SERVICE_UNAVAILABLE log_debug(4, "Connected to 3rd party server:", connection.sock.getpeername()) # Put the request out on the wire. response = None try: # We'll redirect to the URI made in the original request, but with # the new server instead. log_debug(4, "Making request: ", self.req.method, uri) connection.putrequest(self.req.method, uri) # Add some custom headers. if loopProtection: connection.putheader(rhnConstants.HEADER_RHN_REDIRECT, '0') log_debug(4, " Adding original URL header: ", self.rhnParent) connection.putheader(rhnConstants.HEADER_RHN_ORIG_LOC, self.rhnParent) # Add all the other headers in the original request in case we # need to re-authenticate with Hosted. for hdr in list(self.req.headers_in.keys()): if hdr.lower().startswith("x-rhn"): connection.putheader(hdr, self.req.headers_in[hdr]) log_debug(4, "Passing request header: ", hdr, self.req.headers_in[hdr]) connection.endheaders() response = connection.getresponse() except IOError as ioe: # Raised by getresponse() if server closes connection on us. log_error("Redirect connection reset by peer.", redirectLocation, ioe) Traceback(mail=0) # The connection is saved in the current response context, and # will be closed when the caller pops the context. return apache.HTTP_SERVICE_UNAVAILABLE except socket.error as se: # Some socket error occurred. Possibly a read error. log_error("Redirect request failed.", redirectLocation, se) Traceback(mail=0) # The connection is saved in the current response context, and # will be closed when the caller pops the context. return apache.HTTP_SERVICE_UNAVAILABLE # Save the response headers and body FD in the current communication # context. self.responseContext.setBodyFd(response) self.responseContext.setHeaders(response.msg) log_debug(4, "Response headers: ", list(self.responseContext.getHeaders().items())) log_debug(4, "Got redirect response. Status=", response.status) # Return the HTTP status to the caller. return response.status
def __redirectToNextLocationNoRetry(self, loopProtection=False): """ This function will perform a redirection to the next location, as specified in the last response's "Location" header. This function will return an actual HTTP response status code. If successful, it will return apache.HTTP_OK, not apache.OK. If unsuccessful, this function will simply return; no retries will be performed. The following error codes can be returned: HTTP_OK,HTTP_PARTIAL_CONTENT - Redirect successful. HTTP_MOVED_TEMPORARILY - Redirect was redirected again by 3rd party. HTTP_MOVED_PERMANENTLY - Redirect was redirected again by 3rd party. HTTP_INTERNAL_SERVER_ERROR - Error extracting redirect information HTTP_SERVICE_UNAVAILABLE - Could not connect to 3rd party server, connection was reset, or a read error occurred during communication. HTTP_* - Any other HTTP status code may also be returned. Upon successful completion of this function, a new responseContext will be created and pushed onto the stack. """ # Obtain the redirect location first before we replace the current # response context. It's contained in the Location header of the # previous response. redirectLocation = self._get_header(rhnConstants.HEADER_LOCATION) # We are about to redirect to a new location so now we'll push a new # response context before we return any errors. self.responseContext.add() # There should always be a redirect URL passed back to us. If not, # there's an error. if not redirectLocation or len(redirectLocation) == 0: log_error(" No redirect location specified!") Traceback(mail=0) return apache.HTTP_INTERNAL_SERVER_ERROR # The _get_header function returns the value as a list. There should # always be exactly one location specified. redirectLocation = redirectLocation[0] log_debug(1, " Redirecting to: ", redirectLocation) # Tear apart the redirect URL. We need the scheme, the host, the # port (if not the default), and the URI. _scheme, host, port, uri = self._parse_url(redirectLocation) # Add any params onto the URI since _parse_url doesn't include them. if redirectLocation.find('?') > -1: uri += redirectLocation[redirectLocation.index('?'):] # Now create a new connection. We'll use SSL if configured to do # so. params = { 'host': host, 'port': port, } if CFG.has_key('timeout'): params['timeout'] = CFG.TIMEOUT if CFG.USE_SSL: log_debug(1, " Redirecting with SSL. Cert= ", self.caChain) params['trusted_certs'] = [self.caChain] connection = connections.HTTPSConnection(**params) else: log_debug(1, " Redirecting withOUT SSL.") connection = connections.HTTPConnection(**params) # Put the connection into the current response context. self.responseContext.setConnection(connection) # Now open the connection to the 3rd party server. log_debug(4, "Attempting to connect to 3rd party server...") try: connection.connect() except socket.error, e: log_error("Error opening redirect connection", redirectLocation, e) Traceback(mail=0) return apache.HTTP_SERVICE_UNAVAILABLE