def _retry(self, request, reason, spider):
proxy = request.meta.get('proxy')
if proxy:
priority = get_priority(proxy)
if priority:
add_or_update(proxy, priority + PRIORITY_RATE)
proxy = get_proxy()
if proxy:
request.meta['proxy'] = proxy
spider.log('RetryMiddleware used proxy: %s, new proxy: %s' %
(request.meta['proxy'], proxy), logging.DEBUG)
super(ProxyRetryMiddleware, self)._retry(request, reason, spider)
def process_request(self, request, spider):
proxy = get_proxy()
if proxy:
request.meta['proxy'] = proxy