def test_proxy_auth(self): os.environ['http_proxy'] = 'https://*****:*****@proxy:3128' mw = HttpProxyMiddleware() req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjpwYXNz')
def test_proxy_auth(self): os.environ["http_proxy"] = "https://*****:*****@proxy:3128" mw = HttpProxyMiddleware() req = Request("http://scrapytest.org") assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {"proxy": "https://proxy:3128"}) self.assertEquals(req.headers.get("Proxy-Authorization"), "Basic dXNlcjpwYXNz")
def test_no_enviroment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} mw = HttpProxyMiddleware() for url in ('http://e.com', 'https://e.com', 'file:///tmp/a'): req = Request(url) assert mw.process_request(req, spider) is None self.assertEquals(req.url, url) self.assertEquals(req.meta, {})
def test_enviroment_proxies(self): os.environ["http_proxy"] = http_proxy = "https://proxy.for.http:3128" os.environ["https_proxy"] = https_proxy = "http://proxy.for.https:8080" os.environ.pop("file_proxy", None) mw = HttpProxyMiddleware() for url, proxy in [("http://e.com", http_proxy), ("https://e.com", https_proxy), ("file://tmp/a", None)]: req = Request(url) assert mw.process_request(req, spider) is None self.assertEquals(req.url, url) self.assertEquals(req.meta.get("proxy"), proxy)
def test_enviroment_proxies(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' os.environ['https_proxy'] = https_proxy = 'http://proxy.for.https:8080' os.environ.pop('file_proxy', None) mw = HttpProxyMiddleware() for url, proxy in [('http://e.com', http_proxy), ('https://e.com', https_proxy), ('file://tmp/a', None)]: req = Request(url) assert mw.process_request(req, spider) is None self.assertEquals(req.url, url) self.assertEquals(req.meta.get('proxy'), proxy)
def test_no_proxy(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() os.environ['no_proxy'] = '*' req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' not in req.meta os.environ['no_proxy'] = 'other.com' req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' in req.meta os.environ['no_proxy'] = 'other.com,noproxy.com' req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' not in req.meta
def test_no_proxy(self): if sys.version_info < (2, 6): raise SkipTest('no_proxy is not supported in python < 2.6') os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() os.environ['no_proxy'] = '*' req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' not in req.meta os.environ['no_proxy'] = 'other.com' req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' in req.meta os.environ['no_proxy'] = 'other.com,noproxy.com' req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' not in req.meta
def test_no_proxy(self): if sys.version_info < (2, 6): raise SkipTest("no_proxy is not supported in python < 2.6") os.environ["http_proxy"] = http_proxy = "https://proxy.for.http:3128" mw = HttpProxyMiddleware() os.environ["no_proxy"] = "*" req = Request("http://noproxy.com") assert mw.process_request(req, spider) is None assert "proxy" not in req.meta os.environ["no_proxy"] = "other.com" req = Request("http://noproxy.com") assert mw.process_request(req, spider) is None assert "proxy" in req.meta os.environ["no_proxy"] = "other.com,noproxy.com" req = Request("http://noproxy.com") assert mw.process_request(req, spider) is None assert "proxy" not in req.meta
def process_request(self, request, spider): print 'spider name is !!!!!!!!!' + spider.name if spider.name is not 'ohmygourd': # 不需要代理的爬虫直接不使用该组件 try: return HttpProxyMiddleware.process_request(self,request,spider) except AttributeError: return super(SelectorProxyMiddlerware, self).process_request(request, spider) proxy = self.getproxy_ip() if proxy is not None: if proxy['user_pass'] is not None: request.meta['proxy'] = "http://%s" % (proxy['ip_port']) encoded_user_pass = base64.encodestring(proxy['user_pass']).strip() request.headers['Proxy-Authorization'] = 'Basic' + encoded_user_pass else: request.meta['proxy'] = "http://%s" % (proxy['ip_port'])
def test_proxy_already_seted(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() req = Request('http://noproxy.com', meta={'proxy': None}) assert mw.process_request(req, spider) is None assert 'proxy' in req.meta and req.meta['proxy'] is None
def test_proxy_already_seted(self): os.environ["http_proxy"] = http_proxy = "https://proxy.for.http:3128" mw = HttpProxyMiddleware() req = Request("http://noproxy.com", meta={"proxy": None}) assert mw.process_request(req, spider) is None assert "proxy" in req.meta and req.meta["proxy"] is None
def __init__(self): HttpProxyMiddleware.__init__() self.pool = redis.ConnectionPool(max_connections=10, host=settings['REDIS_HOST'], port=settings['REDIS_PORT']) self.redscli = redis.Redis(connection_pool=self.pool)