def get_dict_tweets_list(self, limit, query, url): data_formatter = DataFormatter() dict_tweets_list = [] i = 0 while i < limit: if (i == 0): base_url = url.format(q=query, pos="") else: base_url = url.format(q=query, pos=next_position) url_data = requests.get(base_url, headers=self.HEADER) if url_data.status_code != 200: break content = json.loads(url_data.text) next_position = content['min_position'] html_data = content['items_html'] soup = BeautifulSoup(html_data) tweet_blocks = soup.select('.tweet') for tweet_block in tweet_blocks: if (i < limit): dict_tweets_list.append( data_formatter.get_dict_one_tweet_data(tweet_block)) i += 1 else: break if content['has_more_items'] == False: break return dict_tweets_list