class ProxyMiddleware(object): def __init__(self): self.pl = ProxyList() self.index = 0 self.pl.load_file(os.path.join(BASE_DIR, 'proxy/proxy-list.txt')) def process_request(self, request, spider): request.meta['proxy'] = 'http://{}'.format(self.pl[self.index].address()) print self.pl[self.index] self.index = (self.index + 1) % self.pl.size()
def test_web_proxy_source(self): pl = ProxyList() self.server.response['data'] = DEFAULT_PROXY_LIST_DATA pl.load_url(self.server.get_url()) self.assertEqual(2, pl.size())
def test_file_proxy_source(self): pl = ProxyList() path = self.generate_plist_file(DEFAULT_PROXY_LIST_DATA) pl.load_file(path) self.assertEqual(2, pl.size()) os.unlink(path)
def test_basic(self): pl = ProxyList() self.assertEqual(0, pl.size())