예제 #1
0
파일: main.py 프로젝트: cznyx/zhihu-spider
    def fetch_questions(self):
        questions_list_cursor = get_untouched_question_cursor(self.db, EXPLORE_QUESTION_COUNT)
        for question_item in questions_list_cursor:
            question = question_item['question_id']
            self.update_question_insert_answer(question)

        log.info("Finished crawling {} questions.".format(EXPLORE_QUESTION_COUNT))
예제 #2
0
파일: main.py 프로젝트: ahxxm/zhihu-spider
    def fetch_questions(self):
        questions_list_cursor = get_untouched_question_cursor(self.db, EXPLORE_QUESTION_COUNT)
        question_list = [question_item["question_id"] for question_item in questions_list_cursor]
        tasks = asyncio.wait([self.update_question_insert_answer(question_id)
                              for question_id in question_list])
        self.loop.run_until_complete(tasks)

        log.info("Finished crawling {} questions.".format(EXPLORE_QUESTION_COUNT))