Python lpop Examples

Example #1

File: engine.py Project: fletcher9527/Zhihu_Crawler-1

def gevent_worker(option):
    while True:
        url = red.lpop(red_queue)
        if not url:
            break
        create_new_slave(url, option)

Example #2

File: engine.py Project: 0x24bin/Zhihu_Crawler

def gevent_worker(option):
    while True:
        url = red.lpop(red_queue)
        if not url:
            break
        create_new_slave(url, option)

Example #3

File: engine.py Project: fletcher9527/Zhihu_Crawler-1

    count = 0

    # choose the running way of using database or not

    try:
        option = sys.argv[1]
    except:
        option = ''
    if "mongo" not in option:
        option = "print_data_out"
    i = 0

    # the start page

    red.lpush(red_queue, "https://www.zhihu.com/people/gaoming623")
    url = red.lpop(red_queue)
    create_new_slave(url, option=option)
    for i in range(50):
        url = red.lpop(red_queue)
        create_new_slave(url, option=option)

    process_pool = Pool(multiprocessing.cpu_count() * 2)
    process_pool.map_async(process_worker, option)
    process_pool.close()
    process_pool.join()
    '''
    while(True):

        url_list=[]

        for i in range(50):

Example #4

File: engine.py Project: 0x24bin/Zhihu_Crawler

    count = 0

    # choose the running way of using database or not

    try:
        option = sys.argv[1]
    except:
        option = ''
    if "mongo" not in option:
        option = "print_data_out"
    i = 0

    # the start page

    red.lpush(red_queue, "https://www.zhihu.com/people/gaoming623")
    url = red.lpop(red_queue)
    create_new_slave(url, option=option)
    for i in range(50):
        url = red.lpop(red_queue)
        create_new_slave(url, option=option)

    process_pool = Pool(multiprocessing.cpu_count() * 2)
    process_pool.map_async(process_worker, option)
    process_pool.close()
    process_pool.join()

    '''
    while(True):

        url_list=[]