Esempio n. 1
0
 def start_requests(self):
     yield Request(url=Url.get('stage', {
         'r': self.region,
         't': self.tournament,
         's': self.season,
         'id': self.stage
     }))
Esempio n. 2
0
    def parse(self, response):
        if self.dates is None:
            fixtures = response.xpath(
                '//script[contains(., "DataStore.prime(\'stagefixtures\'")]/text()'
            ).re_first(r"\$\.extend\(.*?\), ([\w\W]*?)\);")
            if fixtures:
                return self.get_fixture(fixtures)
            return

        model_last_mode = response.xpath(
            '//script[contains(., "Model-Last-Mode")]/text()').re_first(
                r"'Model-Last-Mode': '(.*?)' }")
        requests = []

        for d in self.dates.split(','):
            requests.append(
                Request(url=Url.get(
                    'stagefixtures', {
                        'stageId': self.stage,
                        'd': d,
                        'isAggregate': self.is_aggregate
                    }),
                        headers={
                            'X-Requested-With': 'XMLHttpRequest',
                            'Host': 'www.whoscored.com',
                            'Model-Last-Mode': model_last_mode
                        },
                        callback=self.parse_fixtures))

        return requests
Esempio n. 3
0
 def start_requests(self):
     yield Request(url=Url.get(
         'stage', {
             'r': self.region,
             't': self.tournament,
             's': self.season,
             'id': self.stage
         }))
Esempio n. 4
0
    def parse(self, response):
        data = response.xpath('//script[contains(., "var currentTeamId")]/text()') \
            .re_first(r"var currentTeamId = (\d*?);")
        try:
            team_id = int(data)
        except ValueError:
            self.logger.warning("Team id not found: ")
            team_id = None

        dls = response.xpath('//div[@id="player-profile"]//dl')
        name = dls.xpath('dt[contains(., "Name:")]/following-sibling::dd/text()').extract_first()
        full_name = dls.xpath('dt[contains(., "Full Name:")]/following-sibling::dd/text()').extract_first()

        data = dls.xpath('dt[contains(., "Age:")]/following-sibling::dd/text()').re_first(r'(\d+).*')
        try:
            age = int(data)
        except ValueError:
            self.logger.warning("Age not found")
            age = 0

        model_last_mode = response.xpath('//script[contains(., "Model-Last-Mode")]/text()').re_first(
            r"'Model-Last-Mode': '(.*?)' }")

        request = Request(
            url=Url.get('player-statistics', {
                'category': 'summary', 'subcategory': 'all', 'statsAccumulationType': '0', 'isCurrent': 'true',
                'playerId': self.player_id, 'teamIds': '', 'matchId': '', 'stageId': '', 'tournamentOptions': '',
                'sortBy': 'Rating', 'sortAscending': '', 'age': '', 'ageComparisonType': '', 'appearances': '',
                'appearancesComparisonType': '', 'field': 'Overall', 'nationality': '', 'positionOptions': '',
                'timeOfTheGameEnd': '', 'timeOfTheGameStart': '', 'isMinApp': 'false', 'page': '',
                'includeZeroValues': 'true', 'numberOfPlayersToPick': ''}),
            headers={'X-Requested-With': 'XMLHttpRequest', 'Host': 'www.whoscored.com',
                     'Model-Last-Mode': model_last_mode},
            callback=self.parse_player
        )

        request.meta['team_id'] = team_id
        request.meta['name'] = name
        request.meta['full_name'] = full_name
        request.meta['age'] = age

        return request
Esempio n. 5
0
    def parse(self, response):
        if self.dates is None:
            fixtures = response.xpath('//script[contains(., "DataStore.prime(\'stagefixtures\'")]/text()').re_first(
                r"\$\.extend\(.*?\), ([\w\W]*?)\);")
            if fixtures:
                return self.get_fixture(fixtures)
            return

        model_last_mode = response.xpath('//script[contains(., "Model-Last-Mode")]/text()').re_first(
            r"'Model-Last-Mode': '(.*?)' }")
        requests = []

        for d in self.dates.split(','):
            requests.append(Request(
                url=Url.get('stagefixtures', {'stageId': self.stage, 'd': d, 'isAggregate': self.is_aggregate}),
                headers={'X-Requested-With': 'XMLHttpRequest', 'Host': 'www.whoscored.com',
                     'Model-Last-Mode': model_last_mode},
                callback=self.parse_fixtures
            ))

        return requests
Esempio n. 6
0
    def parse(self, response):
        data = response.xpath('//script[contains(., "var currentTeamId")]/text()') \
            .re_first(r"var currentTeamId = (\d*?);")
        try:
            team_id = int(data)
        except ValueError:
            self.logger.warning("Team id not found: ")
            team_id = None

        dls = response.xpath('//div[@id="player-profile"]//dl')
        name = dls.xpath(
            'dt[contains(., "Name:")]/following-sibling::dd/text()'
        ).extract_first()
        full_name = dls.xpath(
            'dt[contains(., "Full Name:")]/following-sibling::dd/text()'
        ).extract_first()

        data = dls.xpath('dt[contains(., "Age:")]/following-sibling::dd/text()'
                         ).re_first(r'(\d+).*')
        try:
            age = int(data)
        except ValueError:
            self.logger.warning("Age not found")
            age = 0

        model_last_mode = response.xpath(
            '//script[contains(., "Model-Last-Mode")]/text()').re_first(
                r"'Model-Last-Mode': '(.*?)' }")

        request = Request(url=Url.get(
            'player-statistics', {
                'category': 'summary',
                'subcategory': 'all',
                'statsAccumulationType': '0',
                'isCurrent': 'true',
                'playerId': self.player_id,
                'teamIds': '',
                'matchId': '',
                'stageId': '',
                'tournamentOptions': '',
                'sortBy': 'Rating',
                'sortAscending': '',
                'age': '',
                'ageComparisonType': '',
                'appearances': '',
                'appearancesComparisonType': '',
                'field': 'Overall',
                'nationality': '',
                'positionOptions': '',
                'timeOfTheGameEnd': '',
                'timeOfTheGameStart': '',
                'isMinApp': 'false',
                'page': '',
                'includeZeroValues': 'true',
                'numberOfPlayersToPick': ''
            }),
                          headers={
                              'X-Requested-With': 'XMLHttpRequest',
                              'Host': 'www.whoscored.com',
                              'Model-Last-Mode': model_last_mode
                          },
                          callback=self.parse_player)

        request.meta['team_id'] = team_id
        request.meta['name'] = name
        request.meta['full_name'] = full_name
        request.meta['age'] = age

        return request