def test_topic_con_pro(self): con_container = Queue() pro_container = Queue() consumer = QueueConsumer(con_container) producer = QueueProducer(pro_container) browser = Browser('firefox') topicconsumer = TopicConsumer(consumer, producer, browser) t_producer = QueueProducer(con_container) topicproducer = TopicProducer(browser, t_producer, wait_login) # 顺便测试了 公用一个browser,是否会出现问题 topicproducer.start() topicconsumer.start() # topicconsumer.start() time.sleep(10) topicconsumer.join() # 所以,落地应该放在store存储这一层,由 consumer producer进行调用 topicproducer.close() topicproducer.stop() # 所以,落地应该放在store存储这一层,由 consumer producer进行调用 topicconsumer.close() topicconsumer.stop()
def test_bstrp(self): # 队列应该可以传送名字,这样,落地的名字就可以很明显了 con_container = Queue() pro_container = Queue() zhcontainer = Queue() consumer = QueueConsumer(con_container) producer = QueueProducer(pro_container) zhfetcher = QueueConsumer(pro_container) store = CSVStore(CSV_FILENAME, CSV_FIELDNAMES) browser = Browser('firefox') # login 和 browser 应该进行封装 # 登录和访问应该使用一个网络,这个网络有多线程池,这样比较好 t_producer = QueueProducer(con_container) topicproducer = TopicProducer(browser, t_producer, wait_login) topicconsumer = TopicConsumer(consumer, producer, browser) zhconsumer = ZHConsumer(zhfetcher, store) topicproducer.start() # time.sleep(10) topicconsumer.start() # time.sleep(10) zhconsumer.start() print("...") zhconsumer.join()
def test_topicconsumer(self): con_container = Queue() pro_container = Queue() consumer = QueueConsumer(con_container) producer = QueueProducer(pro_container) browser = Browser('firefox') # login 和 browser 应该进行封装 # fetcher, pusher, browser wait_login(browser) topicconsumer = TopicConsumer(consumer, producer, browser) for i in range(2): # 应该设计一个resolve url con_container.put("/question/20303645") con_container.put("DIE") topicconsumer.start() while 1: if con_container.qsize() == 0: break else: print("the size of queue : %s" % str(con_container.qsize())) time.sleep(10) # qsize根本用不了啊, 全是html,会很占带宽的 # self.logger.info("长度 : %s " % pro_container) # self.assertEqual(pro_container.qsize(), 10, "获取的信息量有问题") # 所以,落地应该放在store存储这一层,由 consumer producer进行调用 topicconsumer.close() topicconsumer.stop()