class GetData: def __init__(self): self.cookie = GetCookie.login() self.q = Queue() def get_host_main(self, url): html = requests.get(url, cookies=self.cookie).content.decode("utf-8") self.q.html_hosts.append(html) def get_tweet_html(self, url_dic): for i in range(1, int(url_dic['page_amount']) + 1): html = requests.get(url_dic['url'] + str(i), cookies=self.cookie, headers={ "user-agent": random.choice(agents) }).content.decode("utf-8") self.q.add_tweet_html(html) if i > 1: return time.sleep(1) def get_likes_html(self, url_dic): for i in range(1, int(url_dic['page_amount']) + 1): html = requests.get(url_dic["url"] + str(i), cookies=self.cookie, headers={ "user-agent": random.choice(agents) }).content.decode("utf-8") self.q.add_like_html(html) time.sleep(1) def get_comments_html(self, url_dic): for i in range(1, int(url_dic['page_amount']) + 1): print(url_dic['url'] + str(i)) html = requests.get(url_dic["url"] + str(i), cookies=self.cookie, headers={ "user-agent": random.choice(agents) }).content.decode("utf-8") self.q.add_comment_html(html)