class HttpProxyMiddleware(object): def __init__(self, cache_file): if (cache_file is None): cache_file = '/tmp/__proxy_list_cache.json' self.proxies = Proxies(cache_file) @classmethod def from_crawler(cls, crawler): return cls(crawler.settings['PROXY_SERVER_LIST_CACHE_FILE']) def process_request(self, request, spider): proxy = self.proxies.getProxy() print "Using proxy %s"%proxy request.meta['proxy'] = "http://%s" % proxy
class HttpProxyMiddleware(object): def __init__(self, cache_file): if (cache_file is None): cache_file = '/tmp/__proxy_list_cache.json' self.proxies = Proxies(cache_file) @classmethod def from_crawler(cls, crawler): return cls(crawler.settings['PROXY_SERVER_LIST_CACHE_FILE']) def process_request(self, request, spider): proxy = self.proxies.getProxy() print "Using proxy %s" % proxy request.meta['proxy'] = "http://%s" % proxy