def parse_meta_review(self, response):
"""
Crawl on the medias' reviews
sample website: https://www.metacritic.com/game/playstation-4/the-last-of-`us-part-ii/user-reviews?sort-by=score&num_items=100
:param response: the web page
:return: Review Item (type=meta, game id is passed from last method)
"""
print("parse_meta_review.response.url:", response.url)
game_id = response.meta.get("id")
print("game id:", game_id)
all_li = response.xpath("//ol[has-class('critic_reviews')]")[0].xpath(
"li[has-class('critic_review')]")
for li in all_li:
review_section = li.xpath(
"div/div/div/div/div/div[has-class('review_section')]")[0]
review_stats = review_section.xpath(
"div[has-class('review_stats')]")
review_critics = review_stats.xpath(
"div[has-class('review_critic')]")
date = review_critics.xpath("div[has-class('date')]/text()").get()
if date is None:
date = "Not Defined"
print("date:", date)
source = review_critics.xpath(
"div[has-class('source')]/text()").get()
if source is None:
source = review_critics.xpath(
"div[has-class('source')]/a/text()").get()
print("source:", source)
grade = review_stats.xpath(
"div[has-class('review_grade')]/div/text()").get()
grade = int(grade)
print("grade:", grade)
review_body = review_section.xpath(
"div[has-class('review_body')]/text()").get()
review_body_span = review_section.xpath(
"div[has-class('review_body')]/span/text()").get()
review_body_span_expanded = review_section.xpath(
"div[has-class('review_body')]/span/span[has-class("
"'blurb_expanded')]/text()").get()
if review_body_span_expanded is not None:
body = review_body_span_expanded
elif review_body_span is not None:
body = review_body_span
else:
body = review_body
if body is not None:
body = body.strip()
yield reviewItem.Review(type="meta",
score=grade,
date=date,
content=body,
source=source,
game_id=game_id)
def parse_user_review_low_load(self, response):
print("parse_user_review_low_load.response.url:", response.url)
game_id = response.meta.get("id")
print("game id:", game_id)
limit = response.meta.get("limit")
all_li = response.xpath(
"//ol[has-class('user_reviews')]/li[has-class('user_review')]")
for li in all_li:
review_section = li.xpath(
"div/div/div/div/div/div[has-class('review_section')]")[0]
review_stats = review_section.xpath(
"div[has-class('review_stats')]")
review_critics = review_stats.xpath(
"div[has-class('review_critic')]")
date = review_critics.xpath("div[has-class('date')]/text()").get()
if date is None:
date = "Not Defined"
print("date:", date)
source = review_critics.xpath(
"div[has-class('name')]/text()").get()
if source is None or source.strip() == "":
source = review_critics.xpath(
"div[has-class('name')]/a/text()").get()
if source is None or source.strip() == "":
source = review_critics.xpath(
"div[has-class('name')]/span/text()").get()
print("source:", source)
grade = review_stats.xpath(
"div[has-class('review_grade')]/div/text()").get()
grade = float(grade)
print("grade:", grade)
review_body = review_section.xpath(
"div[has-class('review_body')]/text()").get()
review_body_span = review_section.xpath(
"div[has-class('review_body')]/span/text()").get()
review_body_span_expanded = review_section.xpath(
"div[has-class('review_body')]/span/span[has-class("
"'blurb_expanded')]/text()").get()
if review_body_span_expanded is not None:
body = review_body_span_expanded
elif review_body_span is not None:
body = review_body_span
else:
body = review_body
if body is not None:
body = body.strip()
print("body:", body)
if limit != 100:
next_page_url = response.xpath(
"//ul[has-class('pages')]/li[span[has-class('page_num')]]/following-sibling::li[1]/a/@href"
).get()
if next_page_url is not None and next_page_url != "":
yield response.follow(next_page_url,
self.parse_user_review_low_load,
meta={
"limit": limit - 100,
"id": game_id
})
yield reviewItem.Review(type="user",
score=grade,
date=date,
content=body,
source=source,
game_id=game_id)