def test_queue2csvzhconsumer(self):
container = Queue()
consumer = QueueConsumer(container)
store = CSVStore(CSV_FILENAME, CSV_FIELDNAMES)
zhconsumer = ZHConsumer(consumer, store)
browser = Browser('firefox')
browser.visit('http://127.0.0.1:8888')
zhconsumer.start()
"""函数的启动方式
thread.start_new_thread(函数, (参数列表))
"""
data = html.document_fromstring(browser.html)
for i in range(1000):
container.put(data)
while 1:
if not container.qsize():
break
else:
print("the size of queue : %s" % str(container.qsize()))
# tmd , 不加 睡眠，那么，gil 都在 这个循环的手上了
# time.sleep(1)
zhconsumer.close()
zhconsumer.stop()
def test_queue(self):
"""queue的方法并没有进行白盒测试
"""
container = Queue()
with self.assertRaises(QueueEmpty):
container.get()
for i in range(10):
container.put(i)
for i in range(10):
self.assertEqual(container.get(), i, "get should equal the put")
def test_QueueConsuemr(self):
container = Queue()
consumer = QueueConsumer(container)
with self.assertRaises(QueueEmpty):
consumer.fetch()
for i in range(10):
container.put(i)
consumer.close()
self.assertTrue(os.path.exists(os.path.abspath(os.path.join(DATA_DIR, "QueueConsumer" ))))
def test_topicconsumer(self):
con_container = Queue()
pro_container = Queue()
consumer = QueueConsumer(con_container)
producer = QueueProducer(pro_container)
browser = Browser('firefox')
# login 和 browser 应该进行封装
# fetcher, pusher, browser
wait_login(browser)
topicconsumer = TopicConsumer(consumer, producer, browser)
for i in range(2):
# 应该设计一个resolve url
con_container.put("/question/20303645")
con_container.put("DIE")
topicconsumer.start()
while 1:
if con_container.qsize() == 0:
break
else:
print("the size of queue : %s" % str(con_container.qsize()))
time.sleep(10)
# qsize根本用不了啊， 全是html,会很占带宽的
# self.logger.info("长度 ： %s " % pro_container)
# self.assertEqual(pro_container.qsize(), 10, "获取的信息量有问题")
# 所以，落地应该放在store存储这一层，由 consumer producer进行调用
topicconsumer.close()
topicconsumer.stop()