def _change_proxy_and_resend_request(meta_key, message): log.msg("%s" % message, log.INFO) change_proxy(log_msg=True) log.msg("PROXY CHANGED", log.INFO) log.msg("GEO_LOCATION: %s" % request.meta.get('geo_location'), log.INFO) meta_value = request.meta.get(meta_key, 0) request.meta.update({meta_key: meta_value + 1}) if not request.meta.get('%s_time' % meta_key, None): request.meta.update({'%s_time' % meta_key: time.time()}) return Request( url=request.url, meta=request.meta, dont_filter=True, )
def _retry_proxy(self, request, reason, spider): change_proxy(log_msg=True) # time.sleep(1) return RetryMiddleware._retry(self, request, reason, spider)