class GetData:
def __init__(self):
self.cookie = GetCookie.login()
self.q = Queue()
def get_host_main(self, url):
html = requests.get(url, cookies=self.cookie).content.decode("utf-8")
self.q.html_hosts.append(html)
def get_tweet_html(self, url_dic):
for i in range(1, int(url_dic['page_amount']) + 1):
html = requests.get(url_dic['url'] + str(i),
cookies=self.cookie,
headers={
"user-agent": random.choice(agents)
}).content.decode("utf-8")
self.q.add_tweet_html(html)
if i > 1:
return
time.sleep(1)
def get_likes_html(self, url_dic):
for i in range(1, int(url_dic['page_amount']) + 1):
html = requests.get(url_dic["url"] + str(i),
cookies=self.cookie,
headers={
"user-agent": random.choice(agents)
}).content.decode("utf-8")
self.q.add_like_html(html)
time.sleep(1)
def get_comments_html(self, url_dic):
for i in range(1, int(url_dic['page_amount']) + 1):
print(url_dic['url'] + str(i))
html = requests.get(url_dic["url"] + str(i),
cookies=self.cookie,
headers={
"user-agent": random.choice(agents)
}).content.decode("utf-8")
self.q.add_comment_html(html)
