Esempio n. 1
0
        def _change_proxy_and_resend_request(meta_key, message):
            log.msg("%s" % message, log.INFO)
            change_proxy(log_msg=True)
            log.msg("PROXY CHANGED", log.INFO)
            log.msg("GEO_LOCATION: %s" % request.meta.get('geo_location'), log.INFO)

            meta_value = request.meta.get(meta_key, 0)
            request.meta.update({meta_key: meta_value + 1})

            if not request.meta.get('%s_time' % meta_key, None):
                request.meta.update({'%s_time' % meta_key: time.time()})
            return Request(
                url=request.url,
                meta=request.meta,
                dont_filter=True,
            )
Esempio n. 2
0
        def _change_proxy_and_resend_request(meta_key, message):
            log.msg("%s" % message, log.INFO)
            change_proxy(log_msg=True)
            log.msg("PROXY CHANGED", log.INFO)
            log.msg("GEO_LOCATION: %s" % request.meta.get('geo_location'),
                    log.INFO)

            meta_value = request.meta.get(meta_key, 0)
            request.meta.update({meta_key: meta_value + 1})

            if not request.meta.get('%s_time' % meta_key, None):
                request.meta.update({'%s_time' % meta_key: time.time()})
            return Request(
                url=request.url,
                meta=request.meta,
                dont_filter=True,
            )
Esempio n. 3
0
def _retry_proxy(self, request, reason, spider):
    change_proxy(log_msg=True)
    # time.sleep(1)
    return RetryMiddleware._retry(self, request, reason, spider)
Esempio n. 4
0
def _retry_proxy(self, request, reason, spider):
    change_proxy(log_msg=True)
    # time.sleep(1)
    return RetryMiddleware._retry(self, request, reason, spider)