class ProxyMiddleware(object):
def __init__(self):
self.pl = ProxyList()
self.index = 0
self.pl.load_file(os.path.join(BASE_DIR, 'proxy/proxy-list.txt'))
def process_request(self, request, spider):
request.meta['proxy'] = 'http://{}'.format(self.pl[self.index].address())
print self.pl[self.index]
self.index = (self.index + 1) % self.pl.size()
def test_web_proxy_source(self):
pl = ProxyList()
self.server.response['data'] = DEFAULT_PROXY_LIST_DATA
pl.load_url(self.server.get_url())
self.assertEqual(2, pl.size())
def test_file_proxy_source(self):
pl = ProxyList()
path = self.generate_plist_file(DEFAULT_PROXY_LIST_DATA)
pl.load_file(path)
self.assertEqual(2, pl.size())
os.unlink(path)
def test_basic(self):
pl = ProxyList()
self.assertEqual(0, pl.size())
def test_web_proxy_source(self):
pl = ProxyList()
self.server.response['data'] = DEFAULT_PROXY_LIST_DATA
pl.load_url(self.server.get_url())
self.assertEqual(2, pl.size())
def test_file_proxy_source(self):
pl = ProxyList()
path = self.generate_plist_file(DEFAULT_PROXY_LIST_DATA)
pl.load_file(path)
self.assertEqual(2, pl.size())
os.unlink(path)
def test_basic(self):
pl = ProxyList()
self.assertEqual(0, pl.size())
