def start_requests(self):
yield Request(url=Url.get('stage', {
'r': self.region,
't': self.tournament,
's': self.season,
'id': self.stage
}))
def parse(self, response):
if self.dates is None:
fixtures = response.xpath(
'//script[contains(., "DataStore.prime(\'stagefixtures\'")]/text()'
).re_first(r"\$\.extend\(.*?\), ([\w\W]*?)\);")
if fixtures:
return self.get_fixture(fixtures)
return
model_last_mode = response.xpath(
'//script[contains(., "Model-Last-Mode")]/text()').re_first(
r"'Model-Last-Mode': '(.*?)' }")
requests = []
for d in self.dates.split(','):
requests.append(
Request(url=Url.get(
'stagefixtures', {
'stageId': self.stage,
'd': d,
'isAggregate': self.is_aggregate
}),
headers={
'X-Requested-With': 'XMLHttpRequest',
'Host': 'www.whoscored.com',
'Model-Last-Mode': model_last_mode
},
callback=self.parse_fixtures))
return requests
def start_requests(self):
yield Request(url=Url.get(
'stage', {
'r': self.region,
't': self.tournament,
's': self.season,
'id': self.stage
}))
def parse(self, response):
data = response.xpath('//script[contains(., "var currentTeamId")]/text()') \
.re_first(r"var currentTeamId = (\d*?);")
try:
team_id = int(data)
except ValueError:
self.logger.warning("Team id not found: ")
team_id = None
dls = response.xpath('//div[@id="player-profile"]//dl')
name = dls.xpath('dt[contains(., "Name:")]/following-sibling::dd/text()').extract_first()
full_name = dls.xpath('dt[contains(., "Full Name:")]/following-sibling::dd/text()').extract_first()
data = dls.xpath('dt[contains(., "Age:")]/following-sibling::dd/text()').re_first(r'(\d+).*')
try:
age = int(data)
except ValueError:
self.logger.warning("Age not found")
age = 0
model_last_mode = response.xpath('//script[contains(., "Model-Last-Mode")]/text()').re_first(
r"'Model-Last-Mode': '(.*?)' }")
request = Request(
url=Url.get('player-statistics', {
'category': 'summary', 'subcategory': 'all', 'statsAccumulationType': '0', 'isCurrent': 'true',
'playerId': self.player_id, 'teamIds': '', 'matchId': '', 'stageId': '', 'tournamentOptions': '',
'sortBy': 'Rating', 'sortAscending': '', 'age': '', 'ageComparisonType': '', 'appearances': '',
'appearancesComparisonType': '', 'field': 'Overall', 'nationality': '', 'positionOptions': '',
'timeOfTheGameEnd': '', 'timeOfTheGameStart': '', 'isMinApp': 'false', 'page': '',
'includeZeroValues': 'true', 'numberOfPlayersToPick': ''}),
headers={'X-Requested-With': 'XMLHttpRequest', 'Host': 'www.whoscored.com',
'Model-Last-Mode': model_last_mode},
callback=self.parse_player
)
request.meta['team_id'] = team_id
request.meta['name'] = name
request.meta['full_name'] = full_name
request.meta['age'] = age
return request
def parse(self, response):
if self.dates is None:
fixtures = response.xpath('//script[contains(., "DataStore.prime(\'stagefixtures\'")]/text()').re_first(
r"\$\.extend\(.*?\), ([\w\W]*?)\);")
if fixtures:
return self.get_fixture(fixtures)
return
model_last_mode = response.xpath('//script[contains(., "Model-Last-Mode")]/text()').re_first(
r"'Model-Last-Mode': '(.*?)' }")
requests = []
for d in self.dates.split(','):
requests.append(Request(
url=Url.get('stagefixtures', {'stageId': self.stage, 'd': d, 'isAggregate': self.is_aggregate}),
headers={'X-Requested-With': 'XMLHttpRequest', 'Host': 'www.whoscored.com',
'Model-Last-Mode': model_last_mode},
callback=self.parse_fixtures
))
return requests
def parse(self, response):
data = response.xpath('//script[contains(., "var currentTeamId")]/text()') \
.re_first(r"var currentTeamId = (\d*?);")
try:
team_id = int(data)
except ValueError:
self.logger.warning("Team id not found: ")
team_id = None
dls = response.xpath('//div[@id="player-profile"]//dl')
name = dls.xpath(
'dt[contains(., "Name:")]/following-sibling::dd/text()'
).extract_first()
full_name = dls.xpath(
'dt[contains(., "Full Name:")]/following-sibling::dd/text()'
).extract_first()
data = dls.xpath('dt[contains(., "Age:")]/following-sibling::dd/text()'
).re_first(r'(\d+).*')
try:
age = int(data)
except ValueError:
self.logger.warning("Age not found")
age = 0
model_last_mode = response.xpath(
'//script[contains(., "Model-Last-Mode")]/text()').re_first(
r"'Model-Last-Mode': '(.*?)' }")
request = Request(url=Url.get(
'player-statistics', {
'category': 'summary',
'subcategory': 'all',
'statsAccumulationType': '0',
'isCurrent': 'true',
'playerId': self.player_id,
'teamIds': '',
'matchId': '',
'stageId': '',
'tournamentOptions': '',
'sortBy': 'Rating',
'sortAscending': '',
'age': '',
'ageComparisonType': '',
'appearances': '',
'appearancesComparisonType': '',
'field': 'Overall',
'nationality': '',
'positionOptions': '',
'timeOfTheGameEnd': '',
'timeOfTheGameStart': '',
'isMinApp': 'false',
'page': '',
'includeZeroValues': 'true',
'numberOfPlayersToPick': ''
}),
headers={
'X-Requested-With': 'XMLHttpRequest',
'Host': 'www.whoscored.com',
'Model-Last-Mode': model_last_mode
},
callback=self.parse_player)
request.meta['team_id'] = team_id
request.meta['name'] = name
request.meta['full_name'] = full_name
request.meta['age'] = age
return request
