def get_api_hastag_posts(self, response: Response): hashtag = response.json()['data']['hashtag'] url = self.get_url_to_query_next_posts_for_api(hashtag) if url: yield response.follow(url, callback=self.get_api_hastag_posts) posts: list = hashtag['edge_hashtag_to_media']['edges'] for post in posts: yield InstaPostItem(data=post['node']) if post['node']['edge_media_to_comment']['count'] > 30 or post[ 'node']['edge_liked_by']['count'] > 100: yield response.follow(f'/p/{post["node"]["shortcode"]}/', callback=self.post_page_parse)
def parse(self, response: Response, **kwargs): try: js_data = self.get_js_shared_data(response) yield scrapy.FormRequest( self.__login_url, method='POST', callback=self.parse, formdata={ 'username': self.__login, 'enc_password': self.__password }, headers={'X-CSRFToken': js_data['config']['csrf_token']}) except AttributeError as e: if response.json().get('authenticated'): yield response.follow(self.__tag_url, callback=self.first_tag_page_parse)