class ContributorsSpider(scrapy.Spider):
name = 'subscribers'
allowed_domains = ['api.github.com']
def __init__(self, name=None, **kwargs):
super().__init__(name, **kwargs)
self.page = 0
self.data_github_api = DataGithubApi()
self.data_user = DataUser()
self.table_subscribers = TableSubscribers()
@property
def next_page(self) -> int:
self.page = self.page + 1
return self.page
@property
def next_url(self) -> str:
# base_url = f"https://api.github.com/repos/tensorflow/tensorflow/subscribers"
url = f"{self.data_github_api.subscribers_url}?page={self.next_page}"
print(url)
return url
def start_requests(self):
yield scrapy.Request(url=self.next_url, callback=self.parse)
def parse(self, response):
text = response.text
obj = json.loads(text)
for x in obj:
id = x['id']
login = x['login']
site_admin = x['site_admin']
type = x['type']
self.table_subscribers.write(id, login, site_admin, type)
# self.data_user.add_user_login(login)
if len(text) > 2 and text != '[]':
yield scrapy.Request(url=self.next_url, callback=self.parse)
return None
def __init__(self, name=None, **kwargs):
super().__init__(name, **kwargs)
self.page = 0
self.data_github_api = DataGithubApi()
self.data_user = DataUser()
self.table_subscribers = TableSubscribers()