コード例 #1
0
ファイル: contributors.py プロジェクト: testitesti22/scraper
class ContributorsSpider(scrapy.Spider):
    name = 'contributors'
    allowed_domains = ['api.github.com']

    def __init__(self, name=None, **kwargs):
        super().__init__(name, **kwargs)
        self.page = 0
        self.data_github_api = DataGithubApi()
        self.data_user = DataUser()
        self.table_contributors = TableContributors()

    @property
    def next_page(self) -> int:
        self.page = self.page + 1;
        return self.page

    @property
    def next_url(self) -> str:
        return f"{self.data_github_api.contributors_url}?page={self.next_page}"

    def start_requests(self):
        yield scrapy.Request(url=self.next_url, callback=self.parse)

    def parse(self, response):
        text = response.text
        obj = json.loads(text)
        for x in obj:
            id = x['id']
            login = x['login']
            contributions = x['contributions']
            site_admin = x['site_admin']
            type = x['type']
            self.table_contributors.write(id, login, contributions, site_admin, type)
            self.data_user.add_user_login(login)
        if len(text) > 2 and text != '[]':
            yield scrapy.Request(url=self.next_url, callback=self.parse)
        return None
コード例 #2
0
ファイル: subscribers.py プロジェクト: testitesti22/scraper
 def __init__(self, name=None, **kwargs):
     super().__init__(name, **kwargs)
     self.page = 0
     self.data_github_api = DataGithubApi()
     self.data_user = DataUser()
     self.table_subscribers = TableSubscribers()
コード例 #3
0
ファイル: users.py プロジェクト: testitesti22/scraper
 def __init__(self, name=None, **kwargs):
     super().__init__(name, **kwargs)
     self.user_index = -1
     data_user = DataUser()
     self.users_list: List[str] = data_user.user_login_list
     self.table_users = TableUsers()