def process_request(self, request, spider):
redis = RedisQueue('proxy_ip')
if not redis.empty():
proxy_ip = redis.get()
# print proxy_ip
proxy_para = {
'ip_port': proxy_ip,
'user_pass': ''
}
# print proxy_para
if proxy_para['user_pass'] is not None:
request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
print "**************ProxyMiddleware have pass************" + proxy_para['ip_port']
else:
print "**************ProxyMiddleware no pass************" + proxy_para['ip_port']
request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
redis.put(proxy_ip)
def select_ip(num):
redis_list = []
for i in range(num):
redis_list.append(RedisQueue('proxy_ip_%d' % i))
for each in redis_list:
print each.key
label = random.choice(range(num))
while redis_list[label].empty():
label = random.choice(range(num))
proxy_ip = redis_list[label].get()
return proxy_ip, label
def process_request(self, request, spider):
redis = RedisQueue('proxy_ip')
if not redis.empty():
proxy_ip = redis.get()
# print proxy_ip
proxy_para = {'ip_port': proxy_ip, 'user_pass': ''}
# print proxy_para
if proxy_para['user_pass'] is not None:
request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
request.headers[
'Proxy-Authorization'] = 'Basic ' + encoded_user_pass
print "**************ProxyMiddleware have pass************" + proxy_para[
'ip_port']
else:
print "**************ProxyMiddleware no pass************" + proxy_para[
'ip_port']
request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
redis.put(proxy_ip)
from redis_ip import RedisQueue
redis = RedisQueue('proxy_ip')
if not redis.empty():
proxy_ip = redis.get()
print proxy_ip
proxy_para = {'ip_port': proxy_ip, 'user_pass': ''}
print proxy_para
if proxy_para['user_pass'] is not None:
# request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
# encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
# request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
print "**************ProxyMiddleware have pass************" + proxy_para[
'ip_port']
else:
print "**************ProxyMiddleware no pass************" + proxy_para[
'ip_port']
# request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
# {'ip_port': '218.78.210.190:8080', 'user_pass': ''},
redis.put(proxy_ip)
else:
print 'empty'
# request.meta['proxy'] = 'http://%s' % proxy_ip['ip_port']
from redis_ip import RedisQueue
redis = RedisQueue('proxy_ip')
if not redis.empty():
proxy_ip = redis.get()
print proxy_ip
proxy_para = {
'ip_port': proxy_ip,
'user_pass': ''
}
print proxy_para
if proxy_para['user_pass'] is not None:
# request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
# encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
# request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
print "**************ProxyMiddleware have pass************" + proxy_para['ip_port']
else:
print "**************ProxyMiddleware no pass************" + proxy_para['ip_port']
# request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
# {'ip_port': '218.78.210.190:8080', 'user_pass': ''},
redis.put(proxy_ip)
else:
print 'empty'
# request.meta['proxy'] = 'http://%s' % proxy_ip['ip_port']
