def _get_agent(self, request): if request.proxy: _, _, proxy_host, proxy_port, proxy_params = _parse_url_args(request.proxy) scheme = _parse_url_args(request.url)[0] omit_connect_tunnel = proxy_params.find('noconnect') >= 0 if scheme == 'https' and not omit_connect_tunnel: proxy_conf = (proxy_host, proxy_port, request.headers.get('Proxy-Authorization', None)) return TunnelingAgent(reactor, proxy_conf, self.context_factory, self.timeout, self.bind_address, self.pool) else: endpoint = TCP4ClientEndpoint(reactor, proxy_host, proxy_port, self.timeout, self.bind_address) return ProxyAgent(endpoint) return Agent(reactor, self.context_factory, self.timeout, self.bind_address, self.pool)
def test_parse(self): lip = '127.0.0.1' tests = [ ('http://127.0.0.1?c=v&c2=v2#fragment', ('http', lip, lip, 80, '/?c=v&c2=v2')), ('http://127.0.0.1/?c=v&c2=v2#fragment', ('http', lip, lip, 80, '/?c=v&c2=v2')), ('http://127.0.0.1/foo?c=v&c2=v2#frag', ('http', lip, lip, 80, '/foo?c=v&c2=v2')), ('http://127.0.0.1:100?c=v&c2=v2#fragment', ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), ('http://127.0.0.1:100/?c=v&c2=v2#frag', ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), ('http://127.0.0.1:100/foo?c=v&c2=v2#frag', ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')), ('http://127.0.0.1', ('http', lip, lip, 80, '/')), ('http://127.0.0.1/', ('http', lip, lip, 80, '/')), ('http://127.0.0.1/foo', ('http', lip, lip, 80, '/foo')), ('http://127.0.0.1?param=value', ('http', lip, lip, 80, '/?param=value')), ('http://127.0.0.1/?param=value', ('http', lip, lip, 80, '/?param=value')), ('http://127.0.0.1:12345/foo', ('http', lip + ':12345', lip, 12345, '/foo')), ('http://spam:12345/foo', ('http', 'spam:12345', 'spam', 12345, '/foo')), ('http://spam.test.org/foo', ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')), ('https://127.0.0.1/foo', ('https', lip, lip, 443, '/foo')), ('https://127.0.0.1/?param=value', ('https', lip, lip, 443, '/?param=value')), ('https://127.0.0.1:12345/', ('https', lip + ':12345', lip, 12345, '/')), ('http://crawlmitest.org/foo ', ('http', 'crawlmitest.org', 'crawlmitest.org', 80, '/foo')), ('http://egg:7890 ', ('http', 'egg:7890', 'egg', 7890, '/')), ] f = CrawlmiHTPPClientFactory(Request(url='http://github.com/')) for url, test in tests: self.assertEqual(_parse_url_args(url), test, url)
def test_factory_info(self): def _cbFactoryInfo(ingnored_result, factory): self.assertEquals(factory.status, 200) self.assert_(factory.version.startswith('HTTP/')) self.assertEquals(factory.message, 'OK') self.assertEquals(factory.response_headers['content-length'], '10') url = self.get_url('file') factory = CrawlmiHTPPClientFactory(Request(url)) scheme, netloc, host, port, path = _parse_url_args(url) reactor.connectTCP(host, port, factory) return factory.deferred.addCallback(_cbFactoryInfo, factory)