def test_topic_con_pro(self): con_container = Queue() pro_container = Queue() consumer = QueueConsumer(con_container) producer = QueueProducer(pro_container) browser = Browser('firefox') topicconsumer = TopicConsumer(consumer, producer, browser) t_producer = QueueProducer(con_container) topicproducer = TopicProducer(browser, t_producer, wait_login) # 顺便测试了 公用一个browser,是否会出现问题 topicproducer.start() topicconsumer.start() # topicconsumer.start() time.sleep(10) topicconsumer.join() # 所以,落地应该放在store存储这一层,由 consumer producer进行调用 topicproducer.close() topicproducer.stop() # 所以,落地应该放在store存储这一层,由 consumer producer进行调用 topicconsumer.close() topicconsumer.stop()
def test_topic_concurrent_consu(self): con_container = Queue() pro_container = Queue() consumer = QueueConsumer(con_container) producer = QueueProducer(pro_container) browser = Browser('firefox') cons = [] for i in range(3): # topicconsumer = TopicConsumer(consumer, producer, browser) cons.append(TopicConsumer(consumer, producer, browser)) t_producer = QueueProducer(con_container) topicproducer = TopicProducer(browser, t_producer, wait_login) # 顺便测试了 公用一个browser,是否会出现问题 topicproducer.start() for i in range(3): cons[i].start() # topicconsumer.start() time.sleep(10) for i in range(3): cons[i].join() # 所以,落地应该放在store存储这一层,由 consumer producer进行调用 topicproducer.close() topicproducer.stop() # 所以,落地应该放在store存储这一层,由 consumer producer进行调用 for i in range(3): cons[i].close() cons[i].stop()
def test_bstrp(self): # 队列应该可以传送名字,这样,落地的名字就可以很明显了 con_container = Queue() pro_container = Queue() zhcontainer = Queue() consumer = QueueConsumer(con_container) producer = QueueProducer(pro_container) zhfetcher = QueueConsumer(pro_container) store = CSVStore(CSV_FILENAME, CSV_FIELDNAMES) browser = Browser('firefox') # login 和 browser 应该进行封装 # 登录和访问应该使用一个网络,这个网络有多线程池,这样比较好 t_producer = QueueProducer(con_container) topicproducer = TopicProducer(browser, t_producer, wait_login) topicconsumer = TopicConsumer(consumer, producer, browser) zhconsumer = ZHConsumer(zhfetcher, store) topicproducer.start() # time.sleep(10) topicconsumer.start() # time.sleep(10) zhconsumer.start() print("...") zhconsumer.join()
def test_topicproducer(self): container = Queue() producer = QueueProducer(container) browser = Browser('firefox') # login 和 browser 应该进行封装 topicproducer = TopicProducer(browser, producer, wait_login) topicproducer.start() # topicconsumer.start() time.sleep(10) while 1: if container.qsize() == 41: break else: print("the size of queue : %s" % str(container.qsize())) time.sleep(2) # 所以,落地应该放在store存储这一层,由 consumer producer进行调用 topicproducer.close() topicproducer.stop()