Пример #1
0
    def _retry(self, request, reason, spider):
        proxy = request.meta.get('proxy')
        if proxy:
            priority = get_priority(proxy)
            if priority:
                add_or_update(proxy, priority + PRIORITY_RATE)
            proxy = get_proxy()
            if proxy:
                request.meta['proxy'] = proxy
                spider.log('RetryMiddleware used proxy: %s, new proxy: %s' %
                           (request.meta['proxy'], proxy), logging.DEBUG)

        super(ProxyRetryMiddleware, self)._retry(request, reason, spider)
Пример #2
0
 def process_request(self, request, spider):
     proxy = get_proxy()
     if proxy:
         request.meta['proxy'] = proxy