コード例 #1
0
    def create_work_set(self, target_url):
        if target_url in self.task_complete_set:
            return
        result = Match.column(target_url)
        column_id = result.group('column_id')
        content = Http.get_content('https://zhuanlan.zhihu.com/api/columns/' +
                                   column_id)
        if not content:
            return
        raw_info = json.loads(content)
        info = {}
        info['creator_id'] = raw_info['creator']['slug']
        info['creator_hash'] = raw_info['creator']['hash']
        info['creator_sign'] = raw_info['creator']['bio']
        info['creator_name'] = raw_info['creator']['name']
        info['creator_logo'] = raw_info['creator']['avatar'][
            'template'].replace('{id}',
                                raw_info['creator']['avatar']['id']).replace(
                                    '_{size}', '')

        info['column_id'] = raw_info['slug']
        info['name'] = raw_info['name']
        info['logo'] = raw_info['creator']['avatar']['template'].replace(
            '{id}', raw_info['avatar']['id']).replace('_{size}', '')
        info['article'] = raw_info['postsCount']
        info['follower'] = raw_info['followersCount']
        info['description'] = raw_info['description']
        self.info_list.append(info)
        self.task_complete_set.add(target_url)
        detect_url = 'https://zhuanlan.zhihu.com/api/columns/{}/posts?limit=10&offset='.format(
            column_id)
        for i in range(info['article'] / 10 + 1):
            self.work_set.add(detect_url + str(i * 10))
        return
コード例 #2
0
ファイル: worker.py プロジェクト: FengWenPei/ZhihuHelp
    def create_work_set(self, target_url):
        if target_url in self.task_complete_set:
            return
        result = Match.column(target_url)
        self.column_id = result.group('column_id')
        content = Http.get_content('https://zhuanlan.zhihu.com/api/columns/' + self.column_id)
        if not content:
            return
        raw_info = json.loads(content)
        info = {}
        info['creator_id'] = raw_info['creator']['slug']
        info['creator_hash'] = raw_info['creator']['hash']
        info['creator_sign'] = raw_info['creator']['bio']
        info['creator_name'] = raw_info['creator']['name']
        info['creator_logo'] = raw_info['creator']['avatar']['template'].replace('{id}', raw_info['creator']['avatar'][
            'id']).replace('_{size}', '')

        info['column_id'] = raw_info['slug']
        info['name'] = raw_info['name']
        info['logo'] = raw_info['creator']['avatar']['template'].replace('{id}', raw_info['avatar']['id']).replace(
            '_{size}', '')
        info['article'] = raw_info['postsCount']
        info['follower'] = raw_info['followersCount']
        info['description'] = raw_info['description']
        self.info_list.append(info)
        self.task_complete_set.add(target_url)
        detect_url = 'https://zhuanlan.zhihu.com/api/columns/{}/posts?limit=10&offset='.format(self.column_id)
        for i in range(info['article'] / 10 + 1):
            self.work_set.add(detect_url + str(i * 10))
        return
コード例 #3
0
ファイル: worker.py プロジェクト: HiltonWei/zhihu
    def create_work_set(self, target_url):
        if target_url in self.task_complete_set:
            return
        result = Match.column(target_url)
        self.column_id = result.group("column_id")
        content = Http.get_content("https://zhuanlan.zhihu.com/api/columns/" + self.column_id)
        if not content:
            return
        raw_info = json.loads(content)
        info = {}
        info["creator_id"] = raw_info["creator"]["slug"]
        info["creator_hash"] = raw_info["creator"]["hash"]
        info["creator_sign"] = raw_info["creator"]["bio"]
        info["creator_name"] = raw_info["creator"]["name"]
        info["creator_logo"] = (
            raw_info["creator"]["avatar"]["template"]
            .replace("{id}", raw_info["creator"]["avatar"]["id"])
            .replace("_{size}", "")
        )

        info["column_id"] = raw_info["slug"]
        info["name"] = raw_info["name"]
        info["logo"] = (
            raw_info["creator"]["avatar"]["template"].replace("{id}", raw_info["avatar"]["id"]).replace("_{size}", "")
        )
        info["article"] = raw_info["postsCount"]
        info["follower"] = raw_info["followersCount"]
        info["description"] = raw_info["description"]
        self.info_list.append(info)
        self.task_complete_set.add(target_url)
        detect_url = "https://zhuanlan.zhihu.com/api/columns/{}/posts?limit=10&offset=".format(self.column_id)
        for i in range(info["article"] / 10 + 1):
            self.work_set.add(detect_url + str(i * 10))
        return
コード例 #4
0
ファイル: read_list_parser.py プロジェクト: Erik-ly/ZhihuHelp
 def parse_column(command):
     result = Match.column(command)
     column_id = result.group('column_id')
     task = SingleTask()
     task.kind = 'column'
     task.spider.href = 'https://zhuanlan.zhihu.com/{}'.format(column_id)
     task.book.kind = 'column'
     task.book.sql.info = 'select * from ColumnInfo where column_id = "{}" '.format(column_id)
     task.book.sql.question = ''
     task.book.sql.answer = 'select * from Article where column_id = "{}" '.format(column_id)
     return task
コード例 #5
0
ファイル: read_list_parser.py プロジェクト: zuiwan/ZhihuHelp
 def parse_column(command):
     result = Match.column(command)
     column_id = result.group('column_id')
     task = SingleTask()
     task.kind = 'column'
     task.spider.href = 'https://zhuanlan.zhihu.com/{}'.format(
         column_id)
     task.book.kind = 'column'
     task.book.sql.info = 'select * from ColumnInfo where column_id = "{}" '.format(
         column_id)
     task.book.sql.question = ''
     task.book.sql.answer = 'select * from Article where column_id = "{}" '.format(
         column_id)
     return task
コード例 #6
0
 def parse_column(command):
     result = Match.column(command)
     column_id = result.group(u'column_id')
     task = ColumnTask(column_id)
     return task