def testRetriableErrors(self): """Opener automatically retries on socket errors.""" self.mock(timeutil.BackoffTimer, 'sleep', lambda self: None) failures = [] failure = socket.error(errno.ECONNREFUSED, 'Connection refused') class MockConnection(object): def __init__(self, *args, **kwargs): pass def request(self, *args, **kwargs): if failures: try: raise failures.pop(0) except: raise http_error.RequestError(util.SavedException()) else: return MockResponse.OK opener = opener_mod.URLOpener() opener.connectionFactory = MockConnection # Eventually succeeds failures = [failure] * 2 fobj = opener.open('http://nowhere./') # Fails enough to run out of retries failures = [failure] * 3 err = self.assertRaises(socket.error, opener.open, 'http://nowhere./') self.assertEqual(err.args[0], errno.ECONNREFUSED)
def testProxyErrors(self): """Opener automatically retries on gateway errors.""" self.mock(timeutil.BackoffTimer, 'sleep', lambda self: None) badResponse = MockResponse(502, 'Bad Gateway') responses = [] class MockConnection(object): def __init__(self, *args, **kwargs): pass def request(self, *args, **kwargs): return responses.pop(0) opener = opener_mod.URLOpener() opener.connectionFactory = MockConnection # Eventually succeeds responses = [badResponse] * 2 + [MockResponse.OK] fobj = opener.open('http://nowhere./') # Fails enough to run out of retries responses = [badResponse] * 3 + [MockResponse.OK] err = self.assertRaises(socket.error, opener.open, 'http://nowhere./') # Proxy errors are thrown as socket.error(ECONNREFUSED) self.assertEqual(err.args[0], errno.ECONNREFUSED)
def testFatalErrors(self): """Opener does not retry on post-request errors.""" self.mock(timeutil.BackoffTimer, 'sleep', lambda self: None) failures = [] failure = socket.error(errno.ECONNRESET, 'Connection reset by peer') class MockConnection(object): def __init__(self, *args, **kwargs): pass def request(self, *args, **kwargs): if failures: raise failures.pop(0) else: return MockResponse.OK opener = opener_mod.URLOpener() opener.connectionFactory = MockConnection # Eventually succeeds, but we give up immediatey failures = [failure] err = self.assertRaises(socket.error, opener.open, 'http://nowhere./') self.assertEqual(err.args[0], errno.ECONNRESET)
def testProxyBypassNoProxy(self): environ = os.environ.copy() proxyPlain = request.URL("http://*****:*****@host.example.com:1234") proxyConary = request.URL("conary://*****:*****@host.example.com:1234") url = request.URL("http://rpath.com") # We should not hit the proxy at all self.mock(os, 'environ', environ) # All flavors of localhost should be direct proxyMap = proxy_map.ProxyMap() opener = opmod.URLOpener(proxyMap) # Normal hosts will proxy through self.assertEqual(opener._shouldBypass(url, proxyPlain), False) self.assertEqual(opener._shouldBypass(url, proxyConary), False) for h in httputils.LocalHosts: self.assertEqual( opener._shouldBypass(request.URL("http://%s:33" % h), proxyPlain), True) self.assertEqual( opener._shouldBypass(request.URL("http://%s:33" % h), proxyConary), True) # Force direct for everything on HTTP proxies environ['no_proxy'] = "*" self.assertEqual(opener._shouldBypass(url, proxyPlain), True) # environment variable should not affect the selection of conary proxy self.assertEqual(opener._shouldBypass(url, proxyConary), False) # Test that NO_PROXY is also used, not just no_proxy del environ['no_proxy'] self.assertEqual(opener._shouldBypass(url, proxyPlain), False) environ['NO_PROXY'] = "*" self.assertEqual(opener._shouldBypass(url, proxyPlain), True) self.assertEqual(opener._shouldBypass(url, proxyConary), False) # no_proxy takes precedence over NO_PROXY environ['no_proxy'] = "host.com" self.assertEqual(opener._shouldBypass(url, proxyPlain), False) self.assertEqual( opener._shouldBypass(request.URL("http://host.com"), proxyPlain), True) # http://lynx.isc.org/lynx2.8.5/lynx2-8-5/lynx_help/keystrokes/environments.html - comma-separated list of domains (with a trailing, to force empty domain) environ['no_proxy'] = "host.domain.dom, domain1.dom, domain2, " tests = [ ('rpath.com:80', False), ('domain.com:80', False), ('host.domain.dom:80', True), ('hosthost.domain.dom:80', True), ('sub.host.domain.dom:80', True), ('www.domain1.dom:80', True), # Hmm. We will assume the domain is qualified up to ., the domain2 # example probably assumes it matches any domain2 in the domain # search path, but we don't handle that. ('domain2:80', True), ('www.domain2:80', True), ] for h, expected in tests: self.assertEqual( opener._shouldBypass(request.URL('http://' + h), proxyPlain), expected)
def __init__(self, cfg, path, branch=None): self.wms = WmsClient(cfg) self.path = path self.branch = branch self._open = opener.URLOpener(followRedirects=True).open
def __init__(self, cfg): self.cfg = cfg base = URL(cfg.wmsBase) self.wmsUser = base.userpass self.base = base._replace(userpass=None) self.opener = opener.URLOpener(followRedirects=True)