def get_api_hastag_posts(self, response: Response):
hashtag = response.json()['data']['hashtag']
url = self.get_url_to_query_next_posts_for_api(hashtag)
if url:
yield response.follow(url, callback=self.get_api_hastag_posts)
posts: list = hashtag['edge_hashtag_to_media']['edges']
for post in posts:
yield InstaPostItem(data=post['node'])
if post['node']['edge_media_to_comment']['count'] > 30 or post[
'node']['edge_liked_by']['count'] > 100:
yield response.follow(f'/p/{post["node"]["shortcode"]}/',
callback=self.post_page_parse)
def parse(self, response: Response, **kwargs):
try:
js_data = self.get_js_shared_data(response)
yield scrapy.FormRequest(
self.__login_url,
method='POST',
callback=self.parse,
formdata={
'username': self.__login,
'enc_password': self.__password
},
headers={'X-CSRFToken': js_data['config']['csrf_token']})
except AttributeError as e:
if response.json().get('authenticated'):
yield response.follow(self.__tag_url,
callback=self.first_tag_page_parse)
