def dispose_ip(self, proxy_ip, redis_label): redis_list = [] for i in range(REDIS_NUM): redis_list.append(RedisQueue('proxy_ip_%d' % i)) redis_invalid_ip = RedisQueue('invalid_ip') if redis_label == REDIS_NUM - 1: redis_invalid_ip.put(proxy_ip) redis_list[0].put(get_ip()) else: redis_list[redis_label].remove(proxy_ip) redis_list[redis_label + 1].put(proxy_ip) if redis_list[0].empty(): redis_list[0].put(get_ip()) new_redis_label = random.choice(range(REDIS_NUM)) while redis_list[new_redis_label].empty(): new_redis_label = random.choice(range(REDIS_NUM)) new_proxy_ip = redis_list[new_redis_label].get() redis_list[new_redis_label].put(new_proxy_ip) return new_proxy_ip, new_redis_label
def dispose_ip(self, proxy_ip, redis_label): redis_list = [] for i in range(REDIS_NUM): redis_list.append(RedisQueue('proxy_ip_%d' %i)) redis_invalid_ip = RedisQueue('invalid_ip') if redis_label == REDIS_NUM - 1: redis_invalid_ip.put(proxy_ip) redis_list[0].put(get_ip()) else: redis_list[redis_label].remove(proxy_ip) redis_list[redis_label+1].put(proxy_ip) if redis_list[0].empty(): redis_list[0].put(get_ip()) new_redis_label = random.choice(range(REDIS_NUM)) while redis_list[new_redis_label].empty(): new_redis_label = random.choice(range(REDIS_NUM)) new_proxy_ip = redis_list[new_redis_label].get() redis_list[new_redis_label].put(new_proxy_ip) return new_proxy_ip,new_redis_label
def process_request_origin(self, request, spider): redis = RedisQueue('proxy_ip') if not redis.empty(): proxy_ip = redis.get() else: proxy_ip = get_ip() proxy_para = {'ip_port': proxy_ip, 'user_pass': ''} request.meta['proxy'] = "http://%s" % proxy_para['ip_port'] if proxy_para['user_pass'] is not None: encoded_user_pass = base64.encodestring(proxy_para['user_pass']) request.headers[ 'Proxy-Authorization'] = 'Basic ' + encoded_user_pass print "*********************** RedisProxyMiddleware Using proxy ip: %s *****" % proxy_para[ 'ip_port'] redis.put(proxy_ip)
def process_request_origin(self, request, spider): redis = RedisQueue('proxy_ip') if not redis.empty(): proxy_ip = redis.get() else: proxy_ip = get_ip() proxy_para = { 'ip_port': proxy_ip, 'user_pass': '' } request.meta['proxy'] = "http://%s" % proxy_para['ip_port'] if proxy_para['user_pass'] is not None: encoded_user_pass = base64.encodestring(proxy_para['user_pass']) request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass print "*********************** RedisProxyMiddleware Using proxy ip: %s *****" % proxy_para['ip_port'] redis.put(proxy_ip)
def process_exception(self, request, exception, spider): request_ip = request.meta['proxy'] invalid_ip = request_ip.split('//')[1] redis = RedisQueue('proxy_ip') redis_invalid_ip = RedisQueue('invalid_ip') if not redis.empty(): redis.remove(invalid_ip) redis_invalid_ip.put(invalid_ip) print '+++++++++++++++++++++++%s' % exception print '-----------------------removing ip from redis: %s' % invalid_ip new_ip = get_ip() proxy_para = {'ip_port': new_ip, 'user_pass': ''} request.meta['proxy'] = "http://%s" % proxy_para['ip_port'] if proxy_para['user_pass'] is not None: encoded_user_pass = base64.encodestring(proxy_para['user_pass']) request.headers[ 'Proxy-Authorization'] = 'Basic ' + encoded_user_pass print ">>>>>>>>>>>>>>>>>>>>>>>>>>> switch %s to ip: %s *****" % ( invalid_ip, proxy_para['ip_port']) redis.put(new_ip)
def process_exception(self, request, exception, spider): request_ip = request.meta['proxy'] invalid_ip = request_ip.split('//')[1] redis = RedisQueue('proxy_ip') redis_invalid_ip = RedisQueue('invalid_ip') if not redis.empty(): redis.remove(invalid_ip) redis_invalid_ip.put(invalid_ip) print '+++++++++++++++++++++++%s' %exception print '-----------------------removing ip from redis: %s' %invalid_ip new_ip = get_ip() proxy_para = { 'ip_port': new_ip, 'user_pass': '' } request.meta['proxy'] = "http://%s" % proxy_para['ip_port'] if proxy_para['user_pass'] is not None: encoded_user_pass = base64.encodestring(proxy_para['user_pass']) request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass print ">>>>>>>>>>>>>>>>>>>>>>>>>>> switch %s to ip: %s *****" % (invalid_ip,proxy_para['ip_port']) redis.put(new_ip)