Esempio n. 1
0
    def test_queue2csvzhconsumer(self):

        container = Queue()
        consumer =  QueueConsumer(container)

        store = CSVStore(CSV_FILENAME, CSV_FIELDNAMES)

        zhconsumer = ZHConsumer(consumer, store)

        browser = Browser('firefox')
        browser.visit('http://127.0.0.1:8888')       


        zhconsumer.start()
        """函数的启动方式
        thread.start_new_thread(函数, (参数列表))
        """        
        data = html.document_fromstring(browser.html)
        for i in range(1000):
            container.put(data)

        while 1:
            if not container.qsize():
                break
            else:
                print("the size of queue : %s" % str(container.qsize()))
                # tmd , 不加 睡眠,那么,gil 都在 这个循环的手上了
                # time.sleep(1)

        zhconsumer.close()
        zhconsumer.stop()
Esempio n. 2
0
    def test_QueueProducer(self):

        container = Queue()
        producer =  QueueProducer(container)


        for i in range(10):
            producer.push(i)            

        for i in range(10):
            self.assertEqual(i,container.get())
Esempio n. 3
0
    def test_QueueConsuemr(self):

        container = Queue()
        consumer =  QueueConsumer(container)
        with self.assertRaises(QueueEmpty):
            consumer.fetch()

        for i in range(10):
            container.put(i)

        consumer.close()

        self.assertTrue(os.path.exists(os.path.abspath(os.path.join(DATA_DIR, "QueueConsumer" ))))
Esempio n. 4
0
    def test_topicproducer(self):

        container = Queue()
        producer =  QueueProducer(container)
        browser = Browser('firefox')

        # login 和 browser 应该进行封装
        topicproducer = TopicProducer(browser, producer, wait_login)

        topicproducer.start()

        # topicconsumer.start()
        time.sleep(10)
        while 1:
            if container.qsize() == 41:
                break
            else:
                print("the size of queue : %s" % str(container.qsize()))
                time.sleep(2)

 
        # 所以,落地应该放在store存储这一层,由 consumer producer进行调用
        topicproducer.close()
        topicproducer.stop()
Esempio n. 5
0
    def test_queue(self):
        """queue的方法并没有进行白盒测试
        """
        container =  Queue()
        with self.assertRaises(QueueEmpty):
            container.get()

        for i in range(10):
            container.put(i)

        for i in range(10):
            self.assertEqual(container.get(), i, "get should equal the put")
Esempio n. 6
0
    def test_topicconsumer(self):

        con_container = Queue()
        pro_container = Queue()        
        consumer =  QueueConsumer(con_container)
        producer =  QueueProducer(pro_container)
        browser = Browser('firefox')

        # login 和 browser 应该进行封装
        # fetcher, pusher, browser
        wait_login(browser)
        topicconsumer = TopicConsumer(consumer, producer, browser)

        

        for i in range(2):
            # 应该设计一个resolve url
            con_container.put("/question/20303645")
        con_container.put("DIE")


        topicconsumer.start()


        while 1:
            if con_container.qsize() == 0:
                break
            else:
                print("the size of queue : %s" % str(con_container.qsize()))
                time.sleep(10)

        # qsize根本用不了啊,  全是html,会很占带宽的
        # self.logger.info("长度 : %s " % pro_container)
        # self.assertEqual(pro_container.qsize(), 10, "获取的信息量有问题")

         # 所以,落地应该放在store存储这一层,由 consumer producer进行调用
        topicconsumer.close()
        topicconsumer.stop()