def test_proxy_auth(self):
os.environ['http_proxy'] = 'https://*****:*****@proxy:3128'
mw = HttpProxyMiddleware()
req = Request('http://scrapytest.org')
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjpwYXNz')
def test_proxy_auth(self):
os.environ["http_proxy"] = "https://*****:*****@proxy:3128"
mw = HttpProxyMiddleware()
req = Request("http://scrapytest.org")
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta, {"proxy": "https://proxy:3128"})
self.assertEquals(req.headers.get("Proxy-Authorization"), "Basic dXNlcjpwYXNz")
def test_proxy_auth(self):
os.environ['http_proxy'] = 'https://*****:*****@proxy:3128'
mw = HttpProxyMiddleware()
req = Request('http://scrapytest.org')
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEquals(req.headers.get('Proxy-Authorization'),
'Basic dXNlcjpwYXNz')
def test_no_enviroment_proxies(self):
os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
mw = HttpProxyMiddleware()
for url in ('http://e.com', 'https://e.com', 'file:///tmp/a'):
req = Request(url)
assert mw.process_request(req, spider) is None
self.assertEquals(req.url, url)
self.assertEquals(req.meta, {})
def test_no_enviroment_proxies(self):
os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
mw = HttpProxyMiddleware()
for url in ('http://e.com', 'https://e.com', 'file:///tmp/a'):
req = Request(url)
assert mw.process_request(req, spider) is None
self.assertEquals(req.url, url)
self.assertEquals(req.meta, {})
def test_enviroment_proxies(self):
os.environ["http_proxy"] = http_proxy = "https://proxy.for.http:3128"
os.environ["https_proxy"] = https_proxy = "http://proxy.for.https:8080"
os.environ.pop("file_proxy", None)
mw = HttpProxyMiddleware()
for url, proxy in [("http://e.com", http_proxy), ("https://e.com", https_proxy), ("file://tmp/a", None)]:
req = Request(url)
assert mw.process_request(req, spider) is None
self.assertEquals(req.url, url)
self.assertEquals(req.meta.get("proxy"), proxy)
def test_enviroment_proxies(self):
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
os.environ['https_proxy'] = https_proxy = 'http://proxy.for.https:8080'
os.environ.pop('file_proxy', None)
mw = HttpProxyMiddleware()
for url, proxy in [('http://e.com', http_proxy),
('https://e.com', https_proxy), ('file://tmp/a', None)]:
req = Request(url)
assert mw.process_request(req, spider) is None
self.assertEquals(req.url, url)
self.assertEquals(req.meta.get('proxy'), proxy)
def test_enviroment_proxies(self):
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
os.environ['https_proxy'] = https_proxy = 'http://proxy.for.https:8080'
os.environ.pop('file_proxy', None)
mw = HttpProxyMiddleware()
for url, proxy in [('http://e.com', http_proxy),
('https://e.com', https_proxy),
('file://tmp/a', None)]:
req = Request(url)
assert mw.process_request(req, spider) is None
self.assertEquals(req.url, url)
self.assertEquals(req.meta.get('proxy'), proxy)
def test_no_proxy(self):
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
mw = HttpProxyMiddleware()
os.environ['no_proxy'] = '*'
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' not in req.meta
os.environ['no_proxy'] = 'other.com'
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' in req.meta
os.environ['no_proxy'] = 'other.com,noproxy.com'
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' not in req.meta
def test_no_proxy(self):
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
mw = HttpProxyMiddleware()
os.environ['no_proxy'] = '*'
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' not in req.meta
os.environ['no_proxy'] = 'other.com'
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' in req.meta
os.environ['no_proxy'] = 'other.com,noproxy.com'
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' not in req.meta
def test_no_proxy(self):
if sys.version_info < (2, 6):
raise SkipTest('no_proxy is not supported in python < 2.6')
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
mw = HttpProxyMiddleware()
os.environ['no_proxy'] = '*'
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' not in req.meta
os.environ['no_proxy'] = 'other.com'
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' in req.meta
os.environ['no_proxy'] = 'other.com,noproxy.com'
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' not in req.meta
def test_no_proxy(self):
if sys.version_info < (2, 6):
raise SkipTest("no_proxy is not supported in python < 2.6")
os.environ["http_proxy"] = http_proxy = "https://proxy.for.http:3128"
mw = HttpProxyMiddleware()
os.environ["no_proxy"] = "*"
req = Request("http://noproxy.com")
assert mw.process_request(req, spider) is None
assert "proxy" not in req.meta
os.environ["no_proxy"] = "other.com"
req = Request("http://noproxy.com")
assert mw.process_request(req, spider) is None
assert "proxy" in req.meta
os.environ["no_proxy"] = "other.com,noproxy.com"
req = Request("http://noproxy.com")
assert mw.process_request(req, spider) is None
assert "proxy" not in req.meta
def test_no_proxy(self):
if sys.version_info < (2, 6):
raise SkipTest('no_proxy is not supported in python < 2.6')
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
mw = HttpProxyMiddleware()
os.environ['no_proxy'] = '*'
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' not in req.meta
os.environ['no_proxy'] = 'other.com'
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' in req.meta
os.environ['no_proxy'] = 'other.com,noproxy.com'
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' not in req.meta
def process_request(self, request, spider):
print 'spider name is !!!!!!!!!' + spider.name
if spider.name is not 'ohmygourd': # 不需要代理的爬虫直接不使用该组件
try:
return HttpProxyMiddleware.process_request(self,request,spider)
except AttributeError:
return super(SelectorProxyMiddlerware, self).process_request(request, spider)
proxy = self.getproxy_ip()
if proxy is not None:
if proxy['user_pass'] is not None:
request.meta['proxy'] = "http://%s" % (proxy['ip_port'])
encoded_user_pass = base64.encodestring(proxy['user_pass']).strip()
request.headers['Proxy-Authorization'] = 'Basic' + encoded_user_pass
else:
request.meta['proxy'] = "http://%s" % (proxy['ip_port'])
def test_proxy_already_seted(self):
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
mw = HttpProxyMiddleware()
req = Request('http://noproxy.com', meta={'proxy': None})
assert mw.process_request(req, spider) is None
assert 'proxy' in req.meta and req.meta['proxy'] is None
def test_proxy_already_seted(self):
os.environ["http_proxy"] = http_proxy = "https://proxy.for.http:3128"
mw = HttpProxyMiddleware()
req = Request("http://noproxy.com", meta={"proxy": None})
assert mw.process_request(req, spider) is None
assert "proxy" in req.meta and req.meta["proxy"] is None
def __init__(self):
HttpProxyMiddleware.__init__()
self.pool = redis.ConnectionPool(max_connections=10,
host=settings['REDIS_HOST'],
port=settings['REDIS_PORT'])
self.redscli = redis.Redis(connection_pool=self.pool)
def test_proxy_already_seted(self):
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
mw = HttpProxyMiddleware()
req = Request('http://noproxy.com', meta={'proxy': None})
assert mw.process_request(req, spider) is None
assert 'proxy' in req.meta and req.meta['proxy'] is None