Beispiel #1
0
    def test_bstrp(self):
    	# 队列应该可以传送名字,这样,落地的名字就可以很明显了
        con_container = Queue()
        pro_container = Queue()        
        zhcontainer = Queue()

        consumer =  QueueConsumer(con_container)
        producer =  QueueProducer(pro_container)
        zhfetcher =  QueueConsumer(pro_container)
        store = CSVStore(CSV_FILENAME, CSV_FIELDNAMES)


        browser = Browser('firefox')

        # login 和 browser 应该进行封装
        # 登录和访问应该使用一个网络,这个网络有多线程池,这样比较好
        t_producer =  QueueProducer(con_container)
        topicproducer = TopicProducer(browser, t_producer, wait_login)
        topicconsumer = TopicConsumer(consumer, producer, browser)
        zhconsumer = ZHConsumer(zhfetcher, store)


        topicproducer.start()
        # time.sleep(10)
        topicconsumer.start()
        # time.sleep(10)
        zhconsumer.start()


        print("...")
        zhconsumer.join()
Beispiel #2
0
    def test_queue2csvzhconsumer(self):

        container = Queue()
        consumer =  QueueConsumer(container)

        store = CSVStore(CSV_FILENAME, CSV_FIELDNAMES)

        zhconsumer = ZHConsumer(consumer, store)

        browser = Browser('firefox')
        browser.visit('http://127.0.0.1:8888')       


        zhconsumer.start()
        """函数的启动方式
        thread.start_new_thread(函数, (参数列表))
        """        
        data = html.document_fromstring(browser.html)
        for i in range(1000):
            container.put(data)

        while 1:
            if not container.qsize():
                break
            else:
                print("the size of queue : %s" % str(container.qsize()))
                # tmd , 不加 睡眠,那么,gil 都在 这个循环的手上了
                # time.sleep(1)

        zhconsumer.close()
        zhconsumer.stop()