def test_bstrp(self):
# 队列应该可以传送名字，这样，落地的名字就可以很明显了
con_container = Queue()
pro_container = Queue()
zhcontainer = Queue()
consumer = QueueConsumer(con_container)
producer = QueueProducer(pro_container)
zhfetcher = QueueConsumer(pro_container)
store = CSVStore(CSV_FILENAME, CSV_FIELDNAMES)
browser = Browser('firefox')
# login 和 browser 应该进行封装
# 登录和访问应该使用一个网络，这个网络有多线程池，这样比较好
t_producer = QueueProducer(con_container)
topicproducer = TopicProducer(browser, t_producer, wait_login)
topicconsumer = TopicConsumer(consumer, producer, browser)
zhconsumer = ZHConsumer(zhfetcher, store)
topicproducer.start()
# time.sleep(10)
topicconsumer.start()
# time.sleep(10)
zhconsumer.start()
print("...")
zhconsumer.join()
def test_topic_con_pro(self):
con_container = Queue()
pro_container = Queue()
consumer = QueueConsumer(con_container)
producer = QueueProducer(pro_container)
browser = Browser('firefox')
topicconsumer = TopicConsumer(consumer, producer, browser)
t_producer = QueueProducer(con_container)
topicproducer = TopicProducer(browser, t_producer, wait_login)
# 顺便测试了 公用一个browser，是否会出现问题
topicproducer.start()
topicconsumer.start()
# topicconsumer.start()
time.sleep(10)
topicconsumer.join()
# 所以，落地应该放在store存储这一层，由 consumer producer进行调用
topicproducer.close()
topicproducer.stop()
# 所以，落地应该放在store存储这一层，由 consumer producer进行调用
topicconsumer.close()
topicconsumer.stop()
def test_topicconsumer(self):
con_container = Queue()
pro_container = Queue()
consumer = QueueConsumer(con_container)
producer = QueueProducer(pro_container)
browser = Browser('firefox')
# login 和 browser 应该进行封装
# fetcher, pusher, browser
wait_login(browser)
topicconsumer = TopicConsumer(consumer, producer, browser)
for i in range(2):
# 应该设计一个resolve url
con_container.put("/question/20303645")
con_container.put("DIE")
topicconsumer.start()
while 1:
if con_container.qsize() == 0:
break
else:
print("the size of queue : %s" % str(con_container.qsize()))
time.sleep(10)
# qsize根本用不了啊， 全是html,会很占带宽的
# self.logger.info("长度 ： %s " % pro_container)
# self.assertEqual(pro_container.qsize(), 10, "获取的信息量有问题")
# 所以，落地应该放在store存储这一层，由 consumer producer进行调用
topicconsumer.close()
topicconsumer.stop()
