Python RedisQueue.get示例

示例#1

文件： test.py 项目： wacquser/spiders

def process_request(self, request, spider):
    redis = RedisQueue('proxy_ip')
    if not redis.empty():
        proxy_ip = redis.get()
        # print proxy_ip
        proxy_para = {'ip_port': proxy_ip, 'user_pass': ''}
        # print proxy_para
        if proxy_para['user_pass'] is not None:
            request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
            encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
            request.headers[
                'Proxy-Authorization'] = 'Basic ' + encoded_user_pass
            print "**************ProxyMiddleware have pass************" + proxy_para[
                'ip_port']
        else:
            print "**************ProxyMiddleware no pass************" + proxy_para[
                'ip_port']
            request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
        redis.put(proxy_ip)

示例#2

文件： test.py 项目： Kylinlin/spiders

def process_request(self, request, spider):
    redis = RedisQueue('proxy_ip')
    if not redis.empty():
        proxy_ip = redis.get()
        # print proxy_ip
        proxy_para = {
            'ip_port': proxy_ip,
            'user_pass': ''
        }
        # print proxy_para
        if proxy_para['user_pass'] is not None:
            request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
            encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
            request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
            print "**************ProxyMiddleware have pass************" + proxy_para['ip_port']
        else:
            print "**************ProxyMiddleware no pass************" + proxy_para['ip_port']
            request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
        redis.put(proxy_ip)

示例#3

from redis_ip import RedisQueue

redis = RedisQueue('proxy_ip')
if not redis.empty():
    proxy_ip = redis.get()
    print proxy_ip
    proxy_para = {'ip_port': proxy_ip, 'user_pass': ''}
    print proxy_para
    if proxy_para['user_pass'] is not None:
        # request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
        # encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
        # request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
        print "**************ProxyMiddleware have pass************" + proxy_para[
            'ip_port']
    else:
        print "**************ProxyMiddleware no pass************" + proxy_para[
            'ip_port']
        # request.meta['proxy'] = "http://%s" % proxy_para['ip_port']

    # {'ip_port': '218.78.210.190:8080', 'user_pass': ''},
    redis.put(proxy_ip)
else:
    print 'empty'
# request.meta['proxy'] = 'http://%s' % proxy_ip['ip_port']

示例#4

文件： get_ip.py 项目： Kylinlin/spiders

from redis_ip import RedisQueue

redis = RedisQueue('proxy_ip')
if not redis.empty():
    proxy_ip = redis.get()
    print proxy_ip
    proxy_para = {
        'ip_port': proxy_ip,
        'user_pass': ''
    }
    print proxy_para
    if proxy_para['user_pass'] is not None:
        # request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
        # encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
        # request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
        print "**************ProxyMiddleware have pass************" + proxy_para['ip_port']
    else:
        print "**************ProxyMiddleware no pass************" + proxy_para['ip_port']
        # request.meta['proxy'] = "http://%s" % proxy_para['ip_port']

    # {'ip_port': '218.78.210.190:8080', 'user_pass': ''},
    redis.put(proxy_ip)
else:
    print 'empty'
# request.meta['proxy'] = 'http://%s' % proxy_ip['ip_port']