def test_queue2csvzhconsumer(self): container = Queue() consumer = QueueConsumer(container) store = CSVStore(CSV_FILENAME, CSV_FIELDNAMES) zhconsumer = ZHConsumer(consumer, store) browser = Browser('firefox') browser.visit('http://127.0.0.1:8888') zhconsumer.start() """函数的启动方式 thread.start_new_thread(函数, (参数列表)) """ data = html.document_fromstring(browser.html) for i in range(1000): container.put(data) while 1: if not container.qsize(): break else: print("the size of queue : %s" % str(container.qsize())) # tmd , 不加 睡眠,那么,gil 都在 这个循环的手上了 # time.sleep(1) zhconsumer.close() zhconsumer.stop()
def test_topicconsumer(self): con_container = Queue() pro_container = Queue() consumer = QueueConsumer(con_container) producer = QueueProducer(pro_container) browser = Browser('firefox') # login 和 browser 应该进行封装 # fetcher, pusher, browser wait_login(browser) topicconsumer = TopicConsumer(consumer, producer, browser) for i in range(2): # 应该设计一个resolve url con_container.put("/question/20303645") con_container.put("DIE") topicconsumer.start() while 1: if con_container.qsize() == 0: break else: print("the size of queue : %s" % str(con_container.qsize())) time.sleep(10) # qsize根本用不了啊, 全是html,会很占带宽的 # self.logger.info("长度 : %s " % pro_container) # self.assertEqual(pro_container.qsize(), 10, "获取的信息量有问题") # 所以,落地应该放在store存储这一层,由 consumer producer进行调用 topicconsumer.close() topicconsumer.stop()
def test_topicproducer(self): container = Queue() producer = QueueProducer(container) browser = Browser('firefox') # login 和 browser 应该进行封装 topicproducer = TopicProducer(browser, producer, wait_login) topicproducer.start() # topicconsumer.start() time.sleep(10) while 1: if container.qsize() == 41: break else: print("the size of queue : %s" % str(container.qsize())) time.sleep(2) # 所以,落地应该放在store存储这一层,由 consumer producer进行调用 topicproducer.close() topicproducer.stop()