def test_bstrp(self): # 队列应该可以传送名字,这样,落地的名字就可以很明显了 con_container = Queue() pro_container = Queue() zhcontainer = Queue() consumer = QueueConsumer(con_container) producer = QueueProducer(pro_container) zhfetcher = QueueConsumer(pro_container) store = CSVStore(CSV_FILENAME, CSV_FIELDNAMES) browser = Browser('firefox') # login 和 browser 应该进行封装 # 登录和访问应该使用一个网络,这个网络有多线程池,这样比较好 t_producer = QueueProducer(con_container) topicproducer = TopicProducer(browser, t_producer, wait_login) topicconsumer = TopicConsumer(consumer, producer, browser) zhconsumer = ZHConsumer(zhfetcher, store) topicproducer.start() # time.sleep(10) topicconsumer.start() # time.sleep(10) zhconsumer.start() print("...") zhconsumer.join()
def test_queue2csvzhconsumer(self): container = Queue() consumer = QueueConsumer(container) store = CSVStore(CSV_FILENAME, CSV_FIELDNAMES) zhconsumer = ZHConsumer(consumer, store) browser = Browser('firefox') browser.visit('http://127.0.0.1:8888') zhconsumer.start() """函数的启动方式 thread.start_new_thread(函数, (参数列表)) """ data = html.document_fromstring(browser.html) for i in range(1000): container.put(data) while 1: if not container.qsize(): break else: print("the size of queue : %s" % str(container.qsize())) # tmd , 不加 睡眠,那么,gil 都在 这个循环的手上了 # time.sleep(1) zhconsumer.close() zhconsumer.stop()