Python RedisQueueの例

コード例 #1

ファイル: test.py プロジェクト: Kylinlin/spiders

def process_request(self, request, spider):
    redis = RedisQueue('proxy_ip')
    if not redis.empty():
        proxy_ip = redis.get()
        # print proxy_ip
        proxy_para = {
            'ip_port': proxy_ip,
            'user_pass': ''
        }
        # print proxy_para
        if proxy_para['user_pass'] is not None:
            request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
            encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
            request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
            print "**************ProxyMiddleware have pass************" + proxy_para['ip_port']
        else:
            print "**************ProxyMiddleware no pass************" + proxy_para['ip_port']
            request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
        redis.put(proxy_ip)

コード例 #2

ファイル: test.py プロジェクト: wacquser/spiders

def select_ip(num):
    redis_list = []
    for i in range(num):
        redis_list.append(RedisQueue('proxy_ip_%d' % i))
    for each in redis_list:
        print each.key
    label = random.choice(range(num))
    while redis_list[label].empty():
        label = random.choice(range(num))
    proxy_ip = redis_list[label].get()
    return proxy_ip, label

コード例 #3

ファイル: test.py プロジェクト: wacquser/spiders

def process_request(self, request, spider):
    redis = RedisQueue('proxy_ip')
    if not redis.empty():
        proxy_ip = redis.get()
        # print proxy_ip
        proxy_para = {'ip_port': proxy_ip, 'user_pass': ''}
        # print proxy_para
        if proxy_para['user_pass'] is not None:
            request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
            encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
            request.headers[
                'Proxy-Authorization'] = 'Basic ' + encoded_user_pass
            print "**************ProxyMiddleware have pass************" + proxy_para[
                'ip_port']
        else:
            print "**************ProxyMiddleware no pass************" + proxy_para[
                'ip_port']
            request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
        redis.put(proxy_ip)

コード例 #4

from redis_ip import RedisQueue

redis = RedisQueue('proxy_ip')
if not redis.empty():
    proxy_ip = redis.get()
    print proxy_ip
    proxy_para = {'ip_port': proxy_ip, 'user_pass': ''}
    print proxy_para
    if proxy_para['user_pass'] is not None:
        # request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
        # encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
        # request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
        print "**************ProxyMiddleware have pass************" + proxy_para[
            'ip_port']
    else:
        print "**************ProxyMiddleware no pass************" + proxy_para[
            'ip_port']
        # request.meta['proxy'] = "http://%s" % proxy_para['ip_port']

    # {'ip_port': '218.78.210.190:8080', 'user_pass': ''},
    redis.put(proxy_ip)
else:
    print 'empty'
# request.meta['proxy'] = 'http://%s' % proxy_ip['ip_port']

コード例 #5

ファイル: get_ip.py プロジェクト: Kylinlin/spiders

from redis_ip import RedisQueue

redis = RedisQueue('proxy_ip')
if not redis.empty():
    proxy_ip = redis.get()
    print proxy_ip
    proxy_para = {
        'ip_port': proxy_ip,
        'user_pass': ''
    }
    print proxy_para
    if proxy_para['user_pass'] is not None:
        # request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
        # encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
        # request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
        print "**************ProxyMiddleware have pass************" + proxy_para['ip_port']
    else:
        print "**************ProxyMiddleware no pass************" + proxy_para['ip_port']
        # request.meta['proxy'] = "http://%s" % proxy_para['ip_port']

    # {'ip_port': '218.78.210.190:8080', 'user_pass': ''},
    redis.put(proxy_ip)
else:
    print 'empty'
# request.meta['proxy'] = 'http://%s' % proxy_ip['ip_port']