def dispose_ip(self, proxy_ip, redis_label):
redis_list = []
for i in range(REDIS_NUM):
redis_list.append(RedisQueue('proxy_ip_%d' % i))
redis_invalid_ip = RedisQueue('invalid_ip')
if redis_label == REDIS_NUM - 1:
redis_invalid_ip.put(proxy_ip)
redis_list[0].put(get_ip())
else:
redis_list[redis_label].remove(proxy_ip)
redis_list[redis_label + 1].put(proxy_ip)
if redis_list[0].empty():
redis_list[0].put(get_ip())
new_redis_label = random.choice(range(REDIS_NUM))
while redis_list[new_redis_label].empty():
new_redis_label = random.choice(range(REDIS_NUM))
new_proxy_ip = redis_list[new_redis_label].get()
redis_list[new_redis_label].put(new_proxy_ip)
return new_proxy_ip, new_redis_label
def dispose_ip(self, proxy_ip, redis_label):
redis_list = []
for i in range(REDIS_NUM):
redis_list.append(RedisQueue('proxy_ip_%d' %i))
redis_invalid_ip = RedisQueue('invalid_ip')
if redis_label == REDIS_NUM - 1:
redis_invalid_ip.put(proxy_ip)
redis_list[0].put(get_ip())
else:
redis_list[redis_label].remove(proxy_ip)
redis_list[redis_label+1].put(proxy_ip)
if redis_list[0].empty():
redis_list[0].put(get_ip())
new_redis_label = random.choice(range(REDIS_NUM))
while redis_list[new_redis_label].empty():
new_redis_label = random.choice(range(REDIS_NUM))
new_proxy_ip = redis_list[new_redis_label].get()
redis_list[new_redis_label].put(new_proxy_ip)
return new_proxy_ip,new_redis_label
def process_request_origin(self, request, spider):
redis = RedisQueue('proxy_ip')
if not redis.empty():
proxy_ip = redis.get()
else:
proxy_ip = get_ip()
proxy_para = {'ip_port': proxy_ip, 'user_pass': ''}
request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
if proxy_para['user_pass'] is not None:
encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
request.headers[
'Proxy-Authorization'] = 'Basic ' + encoded_user_pass
print "*********************** RedisProxyMiddleware Using proxy ip: %s *****" % proxy_para[
'ip_port']
redis.put(proxy_ip)
def process_request_origin(self, request, spider):
redis = RedisQueue('proxy_ip')
if not redis.empty():
proxy_ip = redis.get()
else:
proxy_ip = get_ip()
proxy_para = {
'ip_port': proxy_ip,
'user_pass': ''
}
request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
if proxy_para['user_pass'] is not None:
encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
print "*********************** RedisProxyMiddleware Using proxy ip: %s *****" % proxy_para['ip_port']
redis.put(proxy_ip)
def process_exception(self, request, exception, spider):
request_ip = request.meta['proxy']
invalid_ip = request_ip.split('//')[1]
redis = RedisQueue('proxy_ip')
redis_invalid_ip = RedisQueue('invalid_ip')
if not redis.empty():
redis.remove(invalid_ip)
redis_invalid_ip.put(invalid_ip)
print '+++++++++++++++++++++++%s' % exception
print '-----------------------removing ip from redis: %s' % invalid_ip
new_ip = get_ip()
proxy_para = {'ip_port': new_ip, 'user_pass': ''}
request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
if proxy_para['user_pass'] is not None:
encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
request.headers[
'Proxy-Authorization'] = 'Basic ' + encoded_user_pass
print ">>>>>>>>>>>>>>>>>>>>>>>>>>> switch %s to ip: %s *****" % (
invalid_ip, proxy_para['ip_port'])
redis.put(new_ip)
def process_exception(self, request, exception, spider):
request_ip = request.meta['proxy']
invalid_ip = request_ip.split('//')[1]
redis = RedisQueue('proxy_ip')
redis_invalid_ip = RedisQueue('invalid_ip')
if not redis.empty():
redis.remove(invalid_ip)
redis_invalid_ip.put(invalid_ip)
print '+++++++++++++++++++++++%s' %exception
print '-----------------------removing ip from redis: %s' %invalid_ip
new_ip = get_ip()
proxy_para = {
'ip_port': new_ip,
'user_pass': ''
}
request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
if proxy_para['user_pass'] is not None:
encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
print ">>>>>>>>>>>>>>>>>>>>>>>>>>> switch %s to ip: %s *****" % (invalid_ip,proxy_para['ip_port'])
redis.put(new_ip)
