def test_bstrp(self):
# 队列应该可以传送名字，这样，落地的名字就可以很明显了
con_container = Queue()
pro_container = Queue()
zhcontainer = Queue()
consumer = QueueConsumer(con_container)
producer = QueueProducer(pro_container)
zhfetcher = QueueConsumer(pro_container)
store = CSVStore(CSV_FILENAME, CSV_FIELDNAMES)
browser = Browser('firefox')
# login 和 browser 应该进行封装
# 登录和访问应该使用一个网络，这个网络有多线程池，这样比较好
t_producer = QueueProducer(con_container)
topicproducer = TopicProducer(browser, t_producer, wait_login)
topicconsumer = TopicConsumer(consumer, producer, browser)
zhconsumer = ZHConsumer(zhfetcher, store)
topicproducer.start()
# time.sleep(10)
topicconsumer.start()
# time.sleep(10)
zhconsumer.start()
print("...")
zhconsumer.join()
def test_queue2csvzhconsumer(self):
container = Queue()
consumer = QueueConsumer(container)
store = CSVStore(CSV_FILENAME, CSV_FIELDNAMES)
zhconsumer = ZHConsumer(consumer, store)
browser = Browser('firefox')
browser.visit('http://127.0.0.1:8888')
zhconsumer.start()
"""函数的启动方式
thread.start_new_thread(函数, (参数列表))
"""
data = html.document_fromstring(browser.html)
for i in range(1000):
container.put(data)
while 1:
if not container.qsize():
break
else:
print("the size of queue : %s" % str(container.qsize()))
# tmd , 不加 睡眠，那么，gil 都在 这个循环的手上了
# time.sleep(1)
zhconsumer.close()
zhconsumer.stop()