def _get_agent(self, request):
if request.proxy:
_, _, proxy_host, proxy_port, proxy_params = _parse_url_args(request.proxy)
scheme = _parse_url_args(request.url)[0]
omit_connect_tunnel = proxy_params.find('noconnect') >= 0
if scheme == 'https' and not omit_connect_tunnel:
proxy_conf = (proxy_host, proxy_port,
request.headers.get('Proxy-Authorization', None))
return TunnelingAgent(reactor, proxy_conf, self.context_factory,
self.timeout, self.bind_address, self.pool)
else:
endpoint = TCP4ClientEndpoint(reactor, proxy_host, proxy_port,
self.timeout, self.bind_address)
return ProxyAgent(endpoint)
return Agent(reactor, self.context_factory, self.timeout, self.bind_address, self.pool)
def _get_agent(self, request):
if request.proxy:
_, _, proxy_host, proxy_port, proxy_params = _parse_url_args(request.proxy)
scheme = _parse_url_args(request.url)[0]
omit_connect_tunnel = proxy_params.find('noconnect') >= 0
if scheme == 'https' and not omit_connect_tunnel:
proxy_conf = (proxy_host, proxy_port,
request.headers.get('Proxy-Authorization', None))
return TunnelingAgent(reactor, proxy_conf, self.context_factory,
self.timeout, self.bind_address, self.pool)
else:
endpoint = TCP4ClientEndpoint(reactor, proxy_host, proxy_port,
self.timeout, self.bind_address)
return ProxyAgent(endpoint)
return Agent(reactor, self.context_factory, self.timeout, self.bind_address, self.pool)
def test_parse(self):
lip = '127.0.0.1'
tests = [
('http://127.0.0.1?c=v&c2=v2#fragment', ('http', lip, lip, 80, '/?c=v&c2=v2')),
('http://127.0.0.1/?c=v&c2=v2#fragment', ('http', lip, lip, 80, '/?c=v&c2=v2')),
('http://127.0.0.1/foo?c=v&c2=v2#frag', ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
('http://127.0.0.1:100?c=v&c2=v2#fragment', ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
('http://127.0.0.1:100/?c=v&c2=v2#frag', ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
('http://127.0.0.1:100/foo?c=v&c2=v2#frag', ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
('http://127.0.0.1', ('http', lip, lip, 80, '/')),
('http://127.0.0.1/', ('http', lip, lip, 80, '/')),
('http://127.0.0.1/foo', ('http', lip, lip, 80, '/foo')),
('http://127.0.0.1?param=value', ('http', lip, lip, 80, '/?param=value')),
('http://127.0.0.1/?param=value', ('http', lip, lip, 80, '/?param=value')),
('http://127.0.0.1:12345/foo', ('http', lip + ':12345', lip, 12345, '/foo')),
('http://spam:12345/foo', ('http', 'spam:12345', 'spam', 12345, '/foo')),
('http://spam.test.org/foo', ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
('https://127.0.0.1/foo', ('https', lip, lip, 443, '/foo')),
('https://127.0.0.1/?param=value', ('https', lip, lip, 443, '/?param=value')),
('https://127.0.0.1:12345/', ('https', lip + ':12345', lip, 12345, '/')),
('http://crawlmitest.org/foo ', ('http', 'crawlmitest.org', 'crawlmitest.org', 80, '/foo')),
('http://egg:7890 ', ('http', 'egg:7890', 'egg', 7890, '/')),
]
f = CrawlmiHTPPClientFactory(Request(url='http://github.com/'))
for url, test in tests:
self.assertEqual(_parse_url_args(url), test, url)
def test_factory_info(self):
def _cbFactoryInfo(ingnored_result, factory):
self.assertEquals(factory.status, 200)
self.assert_(factory.version.startswith('HTTP/'))
self.assertEquals(factory.message, 'OK')
self.assertEquals(factory.response_headers['content-length'], '10')
url = self.get_url('file')
factory = CrawlmiHTPPClientFactory(Request(url))
scheme, netloc, host, port, path = _parse_url_args(url)
reactor.connectTCP(host, port, factory)
return factory.deferred.addCallback(_cbFactoryInfo, factory)
def test_factory_info(self):
def _cbFactoryInfo(ingnored_result, factory):
self.assertEquals(factory.status, 200)
self.assert_(factory.version.startswith('HTTP/'))
self.assertEquals(factory.message, 'OK')
self.assertEquals(factory.response_headers['content-length'], '10')
url = self.get_url('file')
factory = CrawlmiHTPPClientFactory(Request(url))
scheme, netloc, host, port, path = _parse_url_args(url)
reactor.connectTCP(host, port, factory)
return factory.deferred.addCallback(_cbFactoryInfo, factory)
def test_parse(self):
lip = '127.0.0.1'
tests = [
('http://127.0.0.1?c=v&c2=v2#fragment', ('http', lip, lip, 80,
'/?c=v&c2=v2')),
('http://127.0.0.1/?c=v&c2=v2#fragment', ('http', lip, lip, 80,
'/?c=v&c2=v2')),
('http://127.0.0.1/foo?c=v&c2=v2#frag', ('http', lip, lip, 80,
'/foo?c=v&c2=v2')),
('http://127.0.0.1:100?c=v&c2=v2#fragment',
('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
('http://127.0.0.1:100/?c=v&c2=v2#frag',
('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
('http://127.0.0.1:100/foo?c=v&c2=v2#frag',
('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
('http://127.0.0.1', ('http', lip, lip, 80, '/')),
('http://127.0.0.1/', ('http', lip, lip, 80, '/')),
('http://127.0.0.1/foo', ('http', lip, lip, 80, '/foo')),
('http://127.0.0.1?param=value', ('http', lip, lip, 80,
'/?param=value')),
('http://127.0.0.1/?param=value', ('http', lip, lip, 80,
'/?param=value')),
('http://127.0.0.1:12345/foo', ('http', lip + ':12345', lip, 12345,
'/foo')),
('http://spam:12345/foo', ('http', 'spam:12345', 'spam', 12345,
'/foo')),
('http://spam.test.org/foo', ('http', 'spam.test.org',
'spam.test.org', 80, '/foo')),
('https://127.0.0.1/foo', ('https', lip, lip, 443, '/foo')),
('https://127.0.0.1/?param=value', ('https', lip, lip, 443,
'/?param=value')),
('https://127.0.0.1:12345/', ('https', lip + ':12345', lip, 12345,
'/')),
('http://crawlmitest.org/foo ', ('http', 'crawlmitest.org',
'crawlmitest.org', 80, '/foo')),
('http://egg:7890 ', ('http', 'egg:7890', 'egg', 7890, '/')),
]
f = CrawlmiHTPPClientFactory(Request(url='http://github.com/'))
for url, test in tests:
self.assertEqual(_parse_url_args(url), test, url)
