Exemplo n.º 1
0
    def parse_Data(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = take_first(
            response.xpath('//div[@id="event-info"]/'
                           'span[not(@class="more")]/text()').extract())
        dateTime = take_first(dateTime.split(','))
        l.add_value('dateTime', dateTime)

        eventName = take_first(
            response.xpath('//div[@id="event-info"]/'
                           'h1[@class="league-name"]/text()').extract())
        if eventName:
            teams = eventName.lower().split(' - ')
            l.add_value('teams', teams)

        # Markets
        mkts = response.xpath('//div[starts-with(@id, "game-wrapper-s-")]')
        allmktdicts = []
        for mkt in mkts:
            marketName = take_first(mkt.xpath('div/h6/text()').extract())
            mdict = {'marketName': marketName, 'runners': []}
            runners = mkt.xpath('div/ul/li/table/tr/td/form/button')
            for runner in runners:
                runnername = take_first(
                    runner.xpath(
                        'span[@class="option-name"]/text()').extract())
                price = take_first(
                    runner.xpath('span[@class="odds"]/text()').extract())
                mdict['runners'].append({
                    'runnerName': runnername,
                    'price': price
                })
            allmktdicts.append(mdict)

        # Do some Bwin specific post processing and formating
        for mkt in allmktdicts:
            if '3-way - result' == mkt['marketName']:
                mkt['marketName'] = 'Match Odds'
                for runner in mkt['runners']:
                    if teams[0].strip() in runner['runnerName'].lower():
                        runner['runnerName'] = 'HOME'
                    elif teams[1].strip() in runner['runnerName'].lower():
                        runner['runnerName'] = 'AWAY'
                    elif 'X' == runner['runnerName']:
                        runner['runnerName'] = 'DRAW'
            elif 'Goal bet (regular time)' == mkt['marketName']:
                mkt['marketName'] = 'Correct Score'

        # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()
Exemplo n.º 2
0
    def parse_Data(self, response):

        log.msg('Going to parse data for URL: %s' % response.url, level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = response.meta['dateTime']
        l.add_value('dateTime', dateTime)

        eventName = take_first(response.xpath('//div[@class="market_title_top"]/'
                                              'h1/text()').extract())

        if eventName:
            teams = eventName.lower().split(' vs ')
            l.add_value('teams', teams)

        # Markets
        mkts = response.xpath('//div[@class="market_wrapper"]')
        allmktdicts = []
        for mkt in mkts:
            marketName = take_first(mkt.xpath('div[@class="market_title"]/h2/text()').extract())
            mdict = {'marketName': marketName, 'runners': []}
            runners = mkt.xpath('div[@class="market_content_wrapper"]/'
                                'div[@class="market_content"]/table/'
                                'tbody/tr/td[@class="odds"]')
            for runner in runners:
                runnerName = take_first(runner.xpath('a/@title').extract())
                price = take_first(runner.xpath('a/text()').extract())
                if runnerName and price:
                    mdict['runners'].append({'runnerName': runnerName, 'price': price})
            allmktdicts.append(mdict)

        # Do some Totesport specific post processing and formating
        for mkt in allmktdicts:
            if '90 Minutes' == mkt['marketName']:
                mkt['marketName'] = 'Match Odds'
                for runner in mkt['runners']:
                    if teams[0] in runner['runnerName'].lower():
                        runner['runnerName'] = 'HOME'
                    elif teams[1] in runner['runnerName'].lower():
                        runner['runnerName'] = 'AWAY'
                    elif 'Draw' in runner['runnerName']:
                        runner['runnerName'] = 'DRAW'
            elif 'Correct Score' == mkt['marketName']:
                for runner in mkt['runners']:
                    try:
                        if teams[1] in runner['runnerName'].lower():
                            runner['reverse_tag'] = True
                        else:
                            runner['reverse_tag'] = False
                    except AttributeError:
                        print runner
                        continue
        # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()
Exemplo n.º 3
0
    def parseData(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        # JSON is between _jqjsp(....); so extract with regex
        matches = re.findall(r'_jqjsp\((.+?)\);', response.body)
        if matches:
            jsonResp = json.loads(matches[0])

        try:
            jsonData = jsonResp['FetchPrematchEventMarketsByEventIdJSONPResult']
        except KeyError as e:
            log.msg('KeyError: %s' % (self.name, e), level=log.ERROR)
            log.msg('Error response or 404?' % (self.name), level=log.ERROR)
            log.msg('response dump: %s ' % (self.name, jsonResp), level=log.ERROR)
            return []

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = jsonData['DeadlineUTC']
        l.add_value('dateTime', dateTime)

        eventName = jsonData['Name']
        if eventName:
            teams = eventName.lower().split(' - ')
            l.add_value('teams', teams)

        # Markets
        mkts = jsonData['Markets']
        allmktdicts = []
        for mkt in mkts:
            marketName = mkt['Name']
            mdict = {'marketName': marketName, 'runners': []}
            runners = mkt['Selections']
            for runner in runners:
                runnername = runner['Name']
                price = runner['Odds']
                mdict['runners'].append({'runnerName': runnername, 'price': price})
            allmktdicts.append(mdict)

        # Do some Betsafe specific post processing and formating
        for mkt in allmktdicts:
            if mkt['marketName'] == 'Match Winner':
                mkt['marketName'] = 'Match Odds'
                for runner in mkt['runners']:
                    if runner['runnerName'] == u'1':
                        runner['runnerName'] = 'HOME'
                    elif runner['runnerName'] == u'2':
                        runner['runnerName'] = 'AWAY'
                    elif runner['runnerName'] == u'X':
                        runner['runnerName'] = 'DRAW'
        # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()
Exemplo n.º 4
0
    def parseData(self, response):

        jResp = json.loads(response.body)

        try:
            jsonEventsData = take_first(jResp['events'])
            jsonOddsData = jResp['betoffers']
        except KeyError as e:
            log.msg(e, level=log.ERROR)
            log.msg('Error response or 404? ', level=log.ERROR)
            log.msg('Response dump: \n %s' % jResp)
            return []
        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = jsonEventsData['start']
        l.add_value('dateTime', dateTime)

        teams = [jsonEventsData['homeName'], jsonEventsData['awayName']]
        l.add_value('teams', teams)

        # Markets
        allmktdicts = []
        for mkt in jsonOddsData:
            marketName = mkt['criterion']['label']
            mdict = {'marketName': marketName, 'runners': []}
            if 'outcomes' in mkt.keys():
                runners = mkt['outcomes']
                for runner in runners:
                    runnerName = runner['label']
                    if 'line' in runner.keys():
                        # For some reason the over/under markets like 10.5
                        # formatted as 1050 in line field.
                        runnerName += ' %.1f' % (runner['line']/1000)
                    price = runner['oddsFractional']
                    mdict['runners'].append({'runnerName': runnerName, 'price': price})
                allmktdicts.append(mdict)

        # Do some 888 specific post processing and formating
        for mkt in allmktdicts:
            if 'Full Time' == mkt['marketName']:
                mkt['marketName'] = 'Match Odds'
            for runner in mkt['runners']:
                if runner['runnerName'] == u'1':
                    runner['runnerName'] = 'HOME'
                elif runner['runnerName'] == u'2':
                    runner['runnerName'] = 'AWAY'
                elif runner['runnerName'] == u'X':
                    runner['runnerName'] = 'DRAW'

        # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()
Exemplo n.º 5
0
    def parse_Data(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = response.meta['dateTime']
        l.add_value('dateTime', dateTime)

        eventName = response.meta['eventName']
        if eventName:
            teams = eventName.lower().split('-')
            l.add_value('teams', teams)

        # Markets
        MOdict = response.meta['MOdict']
        allmktdicts = [
            MOdict,
        ]
        tables = response.xpath('//table[starts-with(@id, "betTable--")]')
        blockTitle = ''
        block_col_heads = []
        for table in tables:
            title = take_first(
                table.xpath('thead/tr/th[@class="col_title_info"]/'
                            'span/text()').extract())
            col_heads = table.xpath(
                'thead/tr/th[@class="col_bet"]/span/text()').extract()
            if title:
                # Continue to use this title, and update title for block
                blockTitle = title
            else:
                # Use title for block
                title = blockTitle
            if col_heads:
                block_col_heads = col_heads
            else:
                col_heads = block_col_heads
            marketName = take_first(
                table.xpath('tbody/tr/td[@class="col_title"]/'
                            'div/span/a/text()').extract())
            prices = table.xpath(
                'tbody/tr/td[starts-with(@class, "col_bet")]/a/'
                '@data-rate').extract()
            mDict = {'marketName': title + ' ' + marketName, 'runners': []}
            headPrices = zip(col_heads, prices)
            for hP in headPrices:
                mDict['runners'].append({'runnerName': hP[0], 'price': hP[1]})
            allmktdicts.append(mDict)

        # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()
Exemplo n.º 6
0
    def parse_Data(self, response):

        league_name = response.meta['league_name']
        BpEventId = response.meta['BpEventId']

        log.msg('Parsing data for url %s' % response.url)
        log.msg('Parsing for league %s and event Id %s' % (league_name, BpEventId))
        jResp = json.loads(response.body)
        try:
            jData = jResp['response']
        except (TypeError, KeyError):
            jData = jResp['error']
            log.msg('Error response for %s and event Id %s' % (league_name, BpEventId),
                    level=log.ERROR)
            log.msg('Error response: %s' % jData, level=log.ERROR)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        try:
            dateTime = jData['StartTimeUTC']
            l.add_value('dateTime', dateTime)
        except (TypeError, KeyError):
            log.msg('KeyError jData dump is: %s\n KeyError for %s and event Id %s'
                    % (jData, league_name, BpEventId), level=log.ERROR)
            return []

        eventName = jData['name']
        if eventName:
            teams = eventName.lower().split(' v ')
            l.add_value('teams', teams)

        # More ambitiously, grab all markets
        # First all unique market names
        mktNames = set()
        for seln in jData['Selections']:
            mktNames.add(seln['MarketName'])
        # Now construct a market dic for each unique market name
        allMktDicts = []
        for mktName in mktNames:
            mDict = {'marketName': mktName, 'runners': []}
            for seln in jData['Selections']:
                if seln['MarketName'] == mktName:
                    runnerName = seln['BettingAlternativeName'].upper()
                    price = seln['Odds']
                    stake = seln['Stake']
                    mDict['runners'].append({'runnerName': runnerName,
                                             'price': price,
                                             'stake': stake,
                                             })
            allMktDicts.append(mDict)

        # Add markets
        l.add_value('markets', allMktDicts)

        # Load item
        return l.load_item()
Exemplo n.º 7
0
    def parse_Data(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        eventSelection = response.xpath('//markets//market')
        items = []
        for event in eventSelection:

            l = EventLoader(item=EventItem2(), response=response)
            l.add_value('sport', u'Football')
            l.add_value('bookie', self.name)

            dateTime = take_first(event.xpath('tsstart/text()').extract())
            l.add_value('dateTime', dateTime)

            eventName = take_first(event.xpath('eventname/text()').extract())
            if eventName:
                teams = eventName.lower().split(' v ')
                l.add_value('teams', teams)

            MOdict = {'marketName': 'Match Odds', 'runners': []}
            runners = event.xpath('selections//selection')
            for runner in runners:
                runType = runner.xpath('hadvalue/text()').extract()
                odd_up = runner.xpath('currentpriceup/text()').extract()
                odd_down = runner.xpath('currentpricedown/text()').extract()
                if odd_up and odd_down:
                    if runType == [u'H']:
                        MOdict['runners'].append({
                            'runnerName':
                            'HOME',
                            'price':
                            odd_up[0] + '/' + odd_down[0]
                        })
                    elif runType == [u'D']:
                        MOdict['runners'].append({
                            'runnerName':
                            'DRAW',
                            'price':
                            odd_up[0] + '/' + odd_down[0]
                        })
                    elif runType == [u'A']:
                        MOdict['runners'].append({
                            'runnerName':
                            'AWAY',
                            'price':
                            odd_up[0] + '/' + odd_down[0]
                        })

            l.add_value('markets', [
                MOdict,
            ])

            items.append(l.load_item())
        return items
Exemplo n.º 8
0
    def parseData(self, response):
        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = take_first(response.xpath('//hgroup/h1/time/@datetime').extract())
        l.add_value('dateTime', dateTime)

        eventName = response.xpath('//hgroup/h1/text()').extract()
        eventName = ''.join([s.strip() for s in eventName])
        if eventName:
            teams = eventName.lower().split(' - ')
            l.add_value('teams', teams)

        # Markets
        mkts = response.xpath('//div[starts-with(@class, "rollup market_type market_type_id_")]')
        allmktdicts = []
        for mkt in mkts:
            marketName = take_first(mkt.xpath('div[@class="market_type_title"]/h2/'
                                              'text()').extract())
            mdict = {'marketName': marketName, 'runners': []}
            runners = mkt.xpath('.//tr[@class="event"]/td[starts-with(@class, "outcome outcome_")]')
            for runner in runners:
                runnername = take_first(runner.xpath('a/div/span[starts-with(@class, "name")]/text()').extract())
                price = take_first(runner.xpath('a/@data-price-decimal').extract())
                mdict['runners'].append({'runnerName': runnername, 'price': price})
            allmktdicts.append(mdict)

        # Do some Apostas specific post processing and formating
        for mkt in allmktdicts:
            if '1X2 - 90 Min' in mkt['marketName']:
                mkt['marketName'] = 'Match Odds'
                for runner in mkt['runners']:
                    if teams[0] in runner['runnerName'].lower():
                        runner['runnerName'] = 'HOME'
                    elif teams[1] in runner['runnerName'].lower():
                        runner['runnerName'] = 'AWAY'
                    elif 'empate' in runner['runnerName'].lower():
                        runner['runnerName'] = 'DRAW'
            elif u'Resultado exato' in mkt['marketName']:
                mkt['marketName'] = 'Correct Score'
                for runner in mkt['runners']:
                    if teams[1] in runner['runnerName'].lower():
                        runner['reverse_tag'] = True
                    else:
                        runner['reverse_tag'] = False
        # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()
Exemplo n.º 9
0
    def parseData(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        jsonBody = json.loads(response.body)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = jsonBody['startTime']
        l.add_value('dateTime', dateTime)

        eventName = jsonBody['name']
        if eventName:
            teams = eventName.lower().split(' - ')
            if len(teams) != 2:
                teams = eventName.lower().split(' v ')
            l.add_value('teams', teams)

        # Markets
        mkts = jsonBody['children']
        allmktdicts = []
        for mkt in mkts:
            marketName = mkt['name']
            mdict = {'marketName': marketName, 'runners': []}
            runners = mkt['children']
            for runner in runners:
                runnername = runner['name']
                price = runner['odds']
                mdict['runners'].append({
                    'runnerName': runnername,
                    'price': price
                })
            allmktdicts.append(mdict)

        # Do some Skybet specific post processing and formating
        for mkt in allmktdicts:
            if mkt['marketName'] == '1X2':
                mkt['marketName'] = 'Match Odds'
                for runner in mkt['runners']:
                    if teams[0] in runner['runnerName'].lower():
                        runner['runnerName'] = 'HOME'
                    elif teams[1] in runner['runnerName'].lower():
                        runner['runnerName'] = 'AWAY'
                    elif 'Draw' in runner['runnerName']:
                        runner['runnerName'] = 'DRAW'
        # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()
Exemplo n.º 10
0
    def parse_Data(self, response):

        sel = Selector(response)
        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)
        eventSelection = sel.xpath('//markets//market')
        log.msg('Number of events in Selection:  %s .' % len(eventSelection),
                level=log.INFO)

        items = []
        for event in eventSelection:
            l = EventLoader(item=EventItem2(), response=response)
            l.add_value('sport', u'Football')
            l.add_value('bookie', self.name)

            # '2014-03-24T18:00:00'
            dt = event.xpath('tsstart/text()').extract()
            l.add_value('dateTime', dt)

            eventName = take_first(event.xpath('eventname/text()').extract())
            if eventName:
                teams = eventName.lower().split(' v ')
                l.add_value('teams', teams)

            # Markets
            MOdict = {'marketName': 'Match Odds'}
            oddsSelection = event.xpath('selections//selection')
            home_price = draw_price = away_price = 0.00
            for selection in oddsSelection:
                selType = take_first(selection.xpath('hadvalue/text()').extract())
                odd_up = take_first(selection.xpath('currentpriceup/text()').extract())
                odd_down = take_first(selection.xpath('currentpricedown/text()').extract())
                if odd_up and odd_down:
                    if selType == u'H':
                        home_price = odd_up+'/'+odd_down
                    elif selType == u'D':
                        draw_price = odd_up+'/'+odd_down
                    elif selType == u'A':
                        away_price = odd_up+'/'+odd_down
            MOdict['runners'] = [{'runnerName': 'HOME',
                                  'price': home_price},
                                 {'runnerName': 'DRAW',
                                  'price': draw_price},
                                 {'runnerName': 'AWAY',
                                  'price': away_price},
                                 ]

            l.add_value('markets', [MOdict, ])

            items.append(l.load_item())
        return items
Exemplo n.º 11
0
def createMOonly(event, response, bookieName):
    '''
    Take event selection, parse dateTime, teams
    , MO prices then return event loaded.
    (Remember to load item one of the parse methods of spider class
    must return it, but this would terminate current chain, so we pass
    it along in a list, this function just create the item for appending
    to that list)
    '''

    teams = []
    eventName = take_first(event.xpath('td[1]/text()').extract())
    if eventName:
        teams = eventName.lower().split(' - ')
    dateTime = take_first(event.xpath('td[2]/text()').extract())
    home_price = take_first(event.xpath('td[3]/a/text()').extract())
    draw_price = take_first(event.xpath('td[4]/a/text()').extract())
    away_price = take_first(event.xpath('td[5]/a/text()').extract())
    mDict = {'marketName': 'Match Odds',
             'runners': [{'runnerName': 'HOME',
                         'price': home_price},
                         {'runnerName': 'DRAW',
                         'price': draw_price},
                         {'runnerName': 'AWAY',
                         'price': away_price}
                         ]
             }
    # Load the MO data
    l = EventLoader(item=EventItem2(), response=response)
    l.add_value('sport', u'Football')
    l.add_value('bookie', bookieName)
    l.add_value('dateTime', dateTime)
    l.add_value('teams', teams)
    l.add_value('markets', [mDict, ])
    # Debug
    log.msg('No more link for %s, load MO prices only' % eventName, level=log.DEBUG)
    log.msg('Datetime: %s' % dateTime, level=log.DEBUG)
    log.msg('Teams %s' % teams, level=log.DEBUG)
    log.msg('mDict: %s' % mDict, level=log.DEBUG)
    # Return loaded item
    return l.load_item()
Exemplo n.º 12
0
    def parseData(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = take_first(
            response.xpath('//div[@class="eventSubHeader"]/'
                           'span/script/text()').extract())
        dateTime = dateTime[dateTime.find('\"') + 1:dateTime.find('\")')]
        l.add_value('dateTime', dateTime)

        eventName = take_first(
            response.xpath('//div[@id="contentHeader"]/'
                           'h3/text()').extract())
        if eventName:
            teams = eventName.lower().split(' v ')
            l.add_value('teams', teams)

        # Markets
        mkts = response.xpath('//div[starts-with(@id, "mk_")]')
        allmktdicts = []
        for mkt in mkts:
            marketName = take_first(mkt.xpath('ul/li/a/text()').extract())
            mdict = {'marketName': marketName, 'runners': []}
            runners = mkt.xpath('ul/li/ul/li/div[@class="fOutcome"]')
            for runner in runners:
                runnername = take_first(
                    runner.xpath('div[@class="outName"]/text()').extract())
                price = take_first(
                    runner.xpath('div[@class="outBet"]/a/text()').extract())
                mdict['runners'].append({
                    'runnerName': runnername,
                    'price': price
                })
            allmktdicts.append(mdict)

        # Do some Betinternet specific post processing and formating
        for mkt in allmktdicts:
            if '90 mins' in mkt['marketName']:
                mkt['marketName'] = 'Match Odds'
                for runner in mkt['runners']:
                    if teams[0] in runner['runnerName'].lower():
                        runner['runnerName'] = 'HOME'
                    elif teams[1] in runner['runnerName'].lower():
                        runner['runnerName'] = 'AWAY'
                    elif 'Draw' in runner['runnerName']:
                        runner['runnerName'] = 'DRAW'
            elif 'Correct Score' in mkt['marketName']:
                for runner in mkt['runners']:
                    if teams[1] in runner['runnerName'].lower():
                        runner['reverse_tag'] = True
                    else:
                        runner['reverse_tag'] = False
        # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()
Exemplo n.º 13
0
    def parseData(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = take_first(
            response.xpath('//table[@class="liveEventInfo"]/tr/td/'
                           'h1/span[@class="clock"]/text()').extract())
        # Remove the (#686) stuff
        dateTime = dateTime[:dateTime.find('(')]
        l.add_value('dateTime', dateTime)

        eventName = take_first(
            response.xpath('//table[@class="liveEventInfo"]/'
                           'tr/td/h1/text()').extract())
        if eventName:
            teams = eventName.lower().split('vs.')
            l.add_value('teams', teams)

        # Markets
        marketNames = response.xpath(
            '//div[@id="event-filters"]/'
            'h3[@class="capsTableHead"]/text()').extract()
        # Sometimes a single row head gives an extra blank
        marketNames = [mName.strip() for mName in marketNames if mName.strip()]
        marketContent = response.xpath('//div[@id="event-filters"]/table')
        mkts = zip(marketNames, marketContent)
        allmktdicts = []
        for mkt in mkts:
            marketName = mkt[0]
            mdict = {'marketName': marketName, 'runners': []}
            runners = mkt[1].xpath('tr/td')
            for runner in runners:
                runnername = runner.xpath('./text()').extract()
                runnername = take_first(
                    [rName.strip() for rName in runnername if rName.strip()])
                handicapname = take_first(
                    runner.xpath('em[@class="purple"]/text()').extract())
                if handicapname:
                    runnername += ' ' + handicapname.strip()
                price = take_first(runner.xpath('span/text()').extract())
                mdict['runners'].append({
                    'runnerName': runnername,
                    'price': price
                })
            allmktdicts.append(mdict)

        # Do some Tonybet specific post processing and formating
        for mkt in allmktdicts:
            if '2-3 Way' in mkt['marketName']:
                mkt['marketName'] = 'Match Odds'
                for runner in mkt['runners']:
                    if teams[0].strip() in runner['runnerName'].lower():
                        runner['runnerName'] = 'HOME'
                    elif teams[1].strip() in runner['runnerName'].lower():
                        runner['runnerName'] = 'AWAY'
                    elif 'X' == runner['runnerName']:
                        runner['runnerName'] = 'DRAW'
        # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()
Exemplo n.º 14
0
    def parseData(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        # Once against resp is js array
        # jsonstr = re.sub("[,]{2,}", ",", response.body)
        jsonstr = re.sub(",(?=,)", ",null", response.body)
        try:
            jsonBody = json.loads(jsonstr)
        except ValueError as e:
            log.msg(e, level=log.ERROR)
            log.msg('jsonstr: %s' % jsonstr)
            return []

        items = []
        for event in response.meta['events'].values():

            l = EventLoader(item=EventItem2(), response=response)
            l.add_value('sport', u'Football')
            l.add_value('bookie', self.name)

            dateTime = event['dateTime']
            l.add_value('dateTime', dateTime)

            teams = event['teams']
            if teams:
                l.add_value('teams', teams)

            # Get odds
            MOdict = {'marketName': 'Match Odds', 'runners': []}
            for mktData in jsonBody:
                marketId = str(mktData[0])
                # To upgrade in future could be marketData[0] in
                # event['marketIds']
                if marketId == event['MOmarketId']:
                    # Is the assumption 1x2 is first always good?
                    oddsData = mktData[2][0][1]  # 1x2 data is first
                    if oddsData:
                        home_price = am2dec(oddsData[1])
                        draw_price = am2dec(oddsData[3])
                        away_price = am2dec(oddsData[5])
                        MOdict['runners'] = [
                            {
                                'runnerName': 'HOME',
                                'price': home_price
                            },
                            {
                                'runnerName': 'DRAW',
                                'price': draw_price
                            },
                            {
                                'runnerName': 'AWAY',
                                'price': away_price
                            },
                        ]

            # Add markets
            l.add_value('markets', [
                MOdict,
            ])

            # Load item
            items.append(l.load_item())
        return items
Exemplo n.º 15
0
    def parse_Data(self, response):
        '''
        All the data is returned for different markets here, not just 1x2,
        so in theory should be relatively easy to upgrade to more markets
        '''
        # Get the league data we passed as arg through meta
        league_name = response.meta['league_name']

        log.msg('Going to parse data for league: %s' % league_name)

        # Parse JSON from body
        try:
            jResp = json.loads(response.body)
        except ValueError as e:
            log.msg(e, level=log.ERROR)
            log.msg('Response body not JSON for league: %s' % league_name)
            log.msg('response.body dump: \n%s' % response.body)

        # Traverse JSON
        try:
            if 'mod' in jResp.keys():
                jsonMOD = json.loads(jResp['mod'])
            else:
                log.msg("'mod' key not found for league: %s" % league_name)
                log.msg('jResp key dump: %s' % jResp.keys())
                return []
            # Data
            if 'd' in jsonMOD.keys():
                jsonData = jsonMOD['d']
            else:
                log.msg("'d' key not found for league: %s" % league_name)
                log.msg('jsonMOD key dump: %s ' % jsonMOD.keys())
                return []
            # Competitions, it seems if no 1x2 there won't be a c key.
            if 'c' in jsonData.keys():
                jsonComps = jsonData['c']
            else:
                log.msg("'c' key not found for league: %s" % league_name)
                log.msg('jsonData key dump: %s' % jsonData.keys())
                # It looks like jsonMOD['bt'] lists the mkt types avail for sel btn at top,so check with that:
                log.msg("No 1x2 mkt? jsonMOD['bt'] check: %s" % jsonMOD['bt'])
                return []

            # Events (I load the only one comp at once so safe to take zeroth)
            jsonEvents = jsonComps[0]['e']

        except KeyError as e:
            log.msg(e, level=log.ERROR)
            log.msg('Error response or 404? No 1x2 mkts? For league: %s' % league_name)
            return []

        items = []
        for jsonEvent in jsonEvents:

            l = EventLoader(item=EventItem2(), response=response)
            l.add_value('sport', u'Football')
            l.add_value('bookie', self.name)

            dateTime = jsonEvent['i'][4]  # u'09 / 06'
            l.add_value('dateTime', dateTime)

            homeName = jsonEvent['i'][0]
            awayName = jsonEvent['i'][1]
            if homeName and awayName:
                teams = [homeName, awayName]
                l.add_value('teams', teams)
            # MO prices
            MOdict = {'marketName': 'Match Odds'}
            home_price = jsonEvent['o']['1x2'][1]
            draw_price = jsonEvent['o']['1x2'][5]
            away_price = jsonEvent['o']['1x2'][3]
            MOdict['runners'] = [{'runnerName': 'HOME',
                                  'price': home_price},
                                 {'runnerName': 'DRAW',
                                  'price': draw_price},
                                 {'runnerName': 'AWAY',
                                  'price': away_price},
                                 ]

            # Add markets
            l.add_value('markets', [MOdict, ])

            # Load item
            items.append(l.load_item())

        return items
Exemplo n.º 16
0
    def parseData(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        # First row is just league heading,and you want to exclude
        # 'sep' class rows.
        eventSelection = response.xpath(
            '//table[@class="ComingUPTable"]/tr'
            '[position()>1][not(@class="moreEvent")]')
        items = []
        for event in eventSelection:
            l = EventLoader(item=EventItem2(), response=response)
            l.add_value('sport', u'Football')
            l.add_value('bookie', self.name)

            dateTime = take_first(event.xpath('td[2]/text()').extract())
            l.add_value('dateTime', dateTime)

            eventName = take_first(event.xpath('td[3]/div/a/text()').extract())
            if eventName:
                teams = eventName.lower().split(' - ')
                l.add_value('teams', teams)

            # MO prices
            MOdict = {'marketName': 'Match Odds'}

            home_price = take_first(
                event.xpath('td[starts-with(@class, "home")]/'
                            'div[@class="val"]/a/text()').extract())

            draw_price = take_first(
                event.xpath('td[starts-with(@class, "draw")]/'
                            'div[@class="val"]/a/text()').extract())

            away_price = take_first(
                event.xpath('td[starts-with(@class, "away")]/'
                            'div[@class="val"]/a/text()').extract())
            MOdict['runners'] = [
                {
                    'runnerName': 'HOME',
                    'price': home_price
                },
                {
                    'runnerName': 'DRAW',
                    'price': draw_price
                },
                {
                    'runnerName': 'AWAY',
                    'price': away_price
                },
            ]

            # Add markets
            l.add_value('markets', [
                MOdict,
            ])

            # Load item
            items.append(l.load_item())

        return items
Exemplo n.º 17
0
    def parse_Data(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        day = take_first(
            response.xpath(
                '//div[@class="section first"]/div[@class="date"]/'
                'div[@class="cal"]/span[@class="day"]/text()').extract())
        month = take_first(
            response.xpath('//div[@class="section first"]/'
                           'div[@class="date"]/div[@class="cal"]/'
                           'span[@class="month"]/text()').extract())

        if day and month and 'back' not in month:
            # Weirdly sometimes month appears as 'back'
            dateTime = day + ' ' + month
        else:
            # Other format, e.g. Saturday
            dateTime = take_first(
                response.xpath('//div[@class="section first"]/'
                               'div[@class="date"]/text()').extract())

        l.add_value('dateTime', dateTime)

        teams = response.xpath(
            '//div[@id="eventDetailTitle"]/h3/span[@class="name"]/text()'
        ).extract()
        l.add_value('teams', teams)

        # Markets
        mkts = response.xpath('//table[starts-with(@class, "oddTbl")]')
        allmktdicts = []
        for mkt in mkts:
            marketName = take_first(
                mkt.xpath('tr[@class="name"]/th/text()').extract())
            mdict = {'marketName': marketName, 'runners': []}
            if 'Correct Score' in marketName or 'Halftime / Fulltime' in marketName:
                runners = mkt.xpath(
                    'tr[not(starts-with(@class, "result")) and not(@class="name")]/td/a'
                )
                for runner in runners:
                    runnerName = take_first(
                        runner.xpath('span[@class="name"]/text()').extract())
                    price = take_first(
                        runner.xpath('span[@class="odd"]/text()').extract())
                    mdict['runners'].append({
                        'runnerName': runnerName,
                        'price': price
                    })
            else:
                runnerNames = mkt.xpath(
                    'tr[starts-with(@class, "result")]/th/text()').extract()
                if not runnerNames:
                    # Sometimes a span
                    runnerNames = mkt.xpath(
                        'tr[starts-with(@class, "result")]/th/span/text()'
                    ).extract()
                prices = mkt.xpath(
                    'tr[not(starts-with(@class, "result")) and not(@class="name")]/'
                    'td/a/text()').extract()
                runners = zip(runnerNames, prices)
                for runner in runners:
                    mdict['runners'].append({
                        'runnerName': runner[0],
                        'price': runner[1]
                    })
            allmktdicts.append(mdict)

        # Do some Doxxbet specific post processing and formating
        for mkt in allmktdicts:
            if 'Result' == mkt['marketName']:
                mkt['marketName'] = 'Match Odds'
                for runner in mkt['runners']:
                    if runner['runnerName'] == u'1':
                        runner['runnerName'] = 'HOME'
                    elif runner['runnerName'] == u'2':
                        runner['runnerName'] = 'AWAY'
                    elif runner['runnerName'] == u'X':
                        runner['runnerName'] = 'DRAW'
        # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()
Exemplo n.º 18
0
    def parseEvent(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = take_first(
            response.xpath(
                '//div[@class="event-markets"]/div[@class="block-header"]'
                '/div[@class="center-col-options"]'
                '/div[last()]/text()').extract())

        l.add_value('dateTime', dateTime)

        eventName = take_first(
            response.xpath('//div[@class="game-all-markets"]/div/'
                           'div[@class="block-header"]/'
                           'span[@class="block-title"]/text()').extract())

        if eventName:
            teams = eventName.lower().split(' v ')
            l.add_value('teams', teams)

        # I think we need to more specifically target markets formatting too diff
        # between them
        markets = response.xpath('//div[@class="market-wrapper"]')

        # Lets get MO
        # N.B. just like a filesystem path we go upward with '..', after
        # we find the ones with Match Result only
        MOmarket = markets.xpath(
            'div[@class="block-header-wrapper"]/div/'
            'span[@class="block-title" and text()="Match Result"]/'
            '../../../div[@class="matches"]')

        # MO prices
        MOdict = {'marketName': 'Match Odds'}
        home_price = take_first(
            MOmarket.xpath('table/tr[@class="body-row"]/td[1]//'
                           'span[@class="odds-fractional"]/text()').extract())

        draw_price = take_first(
            MOmarket.xpath('table/tr[@class="body-row"]/td[2]//'
                           'span[@class="odds-fractional"]/text()').extract())

        away_price = take_first(
            MOmarket.xpath('table/tr[@class="body-row"]/td[3]//'
                           'span[@class="odds-fractional"]/text()').extract())
        MOdict['runners'] = [
            {
                'runnerName': 'HOME',
                'price': home_price
            },
            {
                'runnerName': 'DRAW',
                'price': draw_price
            },
            {
                'runnerName': 'AWAY',
                'price': away_price
            },
        ]

        # CS market
        CSmarket = markets.xpath(
            'div[@class="block-header-wrapper"]/div/'
            'span[@class="block-title" and text()="Correct Score"]/'
            '../../../div[@class="football-matches"]')
        CSresults = CSmarket.xpath(
            'div[starts-with(@class, "match-row-grouped ")]/'
            'div[@class="match featured-match featured-match-row"]/'
            'div')
        CSdict = {'marketName': 'Correct Score', 'runners': []}
        for result in CSresults:
            runnerName = take_first(
                result.xpath('div[@class="bet-title-3row"]/text()').extract())
            price = take_first(
                result.xpath(
                    'div[@class="home-odds"]/div/'
                    'span[@class="odds-fractional"]/text()').extract())
            if runnerName and price:
                if teams[1] in runnerName.lower():
                    # Tag for score reversing by loader (e.g. if Team2 1-0 want
                    # just 0-1 to match Betfair format and avoid team name comp)
                    CSdict['runners'].append({
                        'runnerName': runnerName,
                        'price': price,
                        'reverse_tag': True
                    })
                else:
                    CSdict['runners'].append({
                        'runnerName': runnerName,
                        'price': price,
                        'reverse_tag': False
                    })

        # Add markets
        l.add_value('markets', [MOdict, CSdict])

        # Load item
        return l.load_item()
Exemplo n.º 19
0
    def parse_Data(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = take_first(
            response.xpath('//div[@class="stats-time"]/'
                           'span[@class="time"]/text()').extract())
        l.add_value('dateTime', dateTime)

        eventName = take_first(
            response.xpath('//div[starts-with(@class, "title ev ev-")]/'
                           'span/text()').extract())
        if eventName:
            teams = eventName.lower().split(' v ')
            l.add_value('teams', teams)

        # Markets (only from main area)
        mkts = response.xpath(
            '//div[@id="main-area"]//div[starts-with(@class, "expander") and @data-mkt_id]'
        )
        allmktdicts = []
        for mkt in mkts:
            marketName = take_first(
                mkt.xpath('h6/span[@class="mkt-name"]/text()').extract())
            if not marketName:
                # Some are not in span
                marketName = take_first(
                    mkt.xpath('h6/text()').extract()).strip()
            mdict = {'marketName': marketName, 'runners': []}
            # N.B. The collapsed markets need seperate AJAX requests to grab data
            # so far now just forget about them (Also sometimes the mkts, e.g. asians
            # are ul/li instead of tables).
            runners = mkt.xpath(
                'div[@class="expander-content"]/table/'
                'tbody/tr[@class="limited-row"]/td[starts-with(@class, "seln")]'
            )
            if not runners:
                # Sometimes tr has no class
                runners = mkt.xpath('div[@class="expander-content"]/table/'
                                    'tbody/tr/td[starts-with(@class, "seln")]')
            for runner in runners:
                runnername = take_first(
                    runner.xpath(
                        './/span[@class="seln-name"]/span/text()').extract())
                if not runnername:
                    # E.g. for Correct Score
                    runnername = take_first(
                        runner.xpath('.//span[@class="seln-sort"]/span/text()'
                                     ).extract())
                price = take_first(
                    runner.xpath(
                        './/span[@class="price dec"]/text()').extract())
                mdict['runners'].append({
                    'runnerName': runnername,
                    'price': price
                })
            allmktdicts.append(mdict)

        # Do some Titanbet specific post processing and formating
        for mkt in allmktdicts:
            if 'Full Time Result' == mkt['marketName']:
                mkt['marketName'] = 'Match Odds'
                for runner in mkt['runners']:
                    if teams[0] in runner['runnerName'].lower():
                        runner['runnerName'] = 'HOME'
                    elif teams[1] in runner['runnerName'].lower():
                        runner['runnerName'] = 'AWAY'
                    elif 'X' == runner['runnerName']:
                        runner['runnerName'] = 'DRAW'
            elif 'Correct Score' in mkt['marketName']:
                for runner in mkt['runners']:
                    if teams[1] in runner['runnerName'].lower():
                        runner['reverse_tag'] = True
                    else:
                        runner['reverse_tag'] = False
        # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()
Exemplo n.º 20
0
    def pre_parse_Data(self, response):

        base_url = 'http://www.fortunawin.com'
        headers = {
            'Host': 'www.fortunawin.com',
            'Referer': response.url,
            'X-Requested-With': 'XMLHttpRequest'
        }
        events = response.xpath('//tr[starts-with(@id, "bet-")]')
        for event in events:
            moreLink = take_first(
                event.xpath(
                    'td[@class="col_toggle"]/'
                    'a[starts-with(@id, "toggler-id-")]/@href').extract())
            dateTime = take_first(
                event.xpath(
                    './/span[starts-with(@id, "datetime-")]/text()').extract())
            dateTime = take_first(dateTime.strip().replace('.',
                                                           '/').rsplit('/', 1))
            eventName = take_first(
                event.xpath('.//td[@class="col_title"]/'
                            'div/span/a/text()').extract())
            # print eventName
            # if 'Antalyaspor' in eventName:
            #    print response.url
            #    stop = raw_input('e2c')
            MOdict = {'marketName': 'Match Odds', 'runners': []}
            home_price = take_first(
                event.xpath('td[2]/a/@data-rate').extract())
            draw_price = take_first(
                event.xpath('td[3]/a/@data-rate').extract())
            away_price = take_first(
                event.xpath('td[4]/a/@data-rate').extract())
            MOdict['runners'] = [
                {
                    'runnerName': 'HOME',
                    'price': home_price
                },
                {
                    'runnerName': 'DRAW',
                    'price': draw_price
                },
                {
                    'runnerName': 'AWAY',
                    'price': away_price
                },
            ]
            eventDic = {
                'eventName': eventName,
                'dateTime': dateTime,
                'MOdict': MOdict
            }
            if moreLink:
                yield Request(url=base_url + moreLink + '&ajax=1',
                              headers=headers,
                              meta=eventDic,
                              callback=self.parse_Data)
            else:
                # Write the MO market alone
                l = EventLoader(item=EventItem2(), response=response)
                l.add_value('sport', u'Football')
                l.add_value('bookie', self.name)
                if eventName:
                    teams = eventName.lower().split('-')
                    l.add_value('teams', teams)
                l.add_value('dateTime', dateTime)
                l.add_value('markets', [
                    MOdict,
                ])
                # Load item
                yield l.load_item()
Exemplo n.º 21
0
    def parse_Data(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)
        # Betway has very annoying display for each league,
        # with all events of certain date under that date block head.
        tableRows = response.xpath('//tbody[@class="oddsbody"]/tr')

        # For all rows under with no date will use latest blockdate,
        # single item list to match xpath extract
        blockdate = []
        items = []
        for row in tableRows:
            # Test with xpath if date row by seeing if td has parent
            # tr with class date. Surely  easier way to just check self?
            if take_first(
                    row.xpath('@class').extract()) and 'date' in take_first(
                        row.xpath('@class').extract()):
                blockdate = row.xpath('td/text()').extract()
            else:
                # Else test if this is an 'event' row, using time as
                # criteria.
                rowtime = row.xpath('td[@class="market_title"]/div[1]/text()'
                                    ).extract()  # 24hr
                if rowtime != []:

                    # We have event.
                    l = EventLoader(item=EventItem2(), response=response)
                    l.add_value('sport', u'Football')
                    l.add_value('bookie', self.name)

                    dateTime = take_first(blockdate)
                    l.add_value('dateTime', dateTime)

                    eventName = take_first(
                        row.xpath('td[@class="market_title"]/'
                                  'a/text()').extract())
                    if eventName:
                        teams = eventName.lower().split(' - ')
                        l.add_value('teams', teams)

                    # MO prices
                    MOdict = {'marketName': 'Match Odds'}
                    home_price = take_first(
                        row.xpath(
                            'td[contains(@class,"outcome")][1]/'
                            'a[@class="outcome"]/'
                            'div[@class="outcome_button"]/text()').extract())

                    draw_price = take_first(
                        row.xpath(
                            'td[contains(@class,"outcome")][2]/'
                            'a[@class="outcome"]/'
                            'div[@class="outcome_button"]/text()').extract())

                    away_price = take_first(
                        row.xpath(
                            'td[contains(@class,"outcome")][3]/'
                            'a[@class="outcome"]/'
                            'div[@class="outcome_button"]/text()').extract())
                    MOdict['runners'] = [
                        {
                            'runnerName': 'HOME',
                            'price': home_price
                        },
                        {
                            'runnerName': 'DRAW',
                            'price': draw_price
                        },
                        {
                            'runnerName': 'AWAY',
                            'price': away_price
                        },
                    ]
                    # Add markets
                    l.add_value('markets', [
                        MOdict,
                    ])

                    # Load item
                    items.append(l.load_item())

        return items
Exemplo n.º 22
0
    def parse_match(self, response):
        eventSelection = response.xpath('//table[@class="tableData"]/tbody/tr')

        if (response.url.endswith('Outright.html')
                or response.url.endswith('Trophy.html')):
            # Not interested, bail.
            # Can't filter this earlier, because of redirect name change of link.
            return []

        items = []
        for event in eventSelection:
            log.msg('Going to parse data for URL: %s' % response.url[20:],
                    level=log.INFO)

            l = EventLoader(item=EventItem2(), response=response)
            l.add_value('sport', u'Football')
            l.add_value('bookie', self.name)

            if response.meta.get('redirect_times') is not None:
                # WilliamHill redirects for markets with single event, must handle differently.
                log.msg('Scraping redirected site: %s' % response.url)

                dateTime = take_first(
                    response.xpath('//div[@id="contentHead"]/'
                                   'span[@id="eventDetailsHeader"]/nobr/'
                                   'span/text()').extract())
                if not dateTime:
                    dateTime = ''
                l.add_value('dateTime', dateTime)

                team1 = take_first(
                    event.xpath('td[1]/div/div[@class="eventselection"]/text()'
                                ).extract())
                team2 = take_first(
                    event.xpath('td[3]/div/div[@class="eventselection"]/text()'
                                ).extract())
                if team1 and team2:
                    l.add_value('teams', [team1, team2])

            else:
                dateTime = take_first(
                    event.xpath('td[1]/span/text()|td[1]/text()').extract())
                if not dateTime:
                    # Could be Live
                    dateTime = ''
                l.add_value('dateTime', dateTime)
                eventName = take_first(
                    event.xpath('td[3]/a/span/text()').extract())
                if eventName:
                    teams = eventName.lower().split(' v ')
                    l.add_value('teams', teams)

            # MO market is same either way
            MOdict = {'marketName': 'Match Odds'}
            home_price = take_first(
                event.xpath(
                    'td[5]/div/div[@class="eventprice"]/text()').extract())
            draw_price = take_first(
                event.xpath(
                    'td[6]/div/div[@class="eventprice"]/text()').extract())
            away_price = take_first(
                event.xpath(
                    'td[7]/div/div[@class="eventprice"]/text()').extract())
            MOdict['runners'] = [
                {
                    'runnerName': 'HOME',
                    'price': home_price
                },
                {
                    'runnerName': 'DRAW',
                    'price': draw_price
                },
                {
                    'runnerName': 'AWAY',
                    'price': away_price
                },
            ]
            # Add markets
            l.add_value('markets', [
                MOdict,
            ])

            # Load item
            items.append(l.load_item())
        return items
Exemplo n.º 23
0
    def parseEvent(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = take_first(
            response.xpath('//table[starts-with(@id, "TBL_Content")]/'
                           'tr[1]/td[@class="playtime"]/p/text()').extract())

        l.add_value('dateTime', dateTime)

        eventName = take_first(
            response.xpath('//td[@class="colmiddle"]/div[@class="bets shadow"]'
                           '/h2[@class="header gradient"]/'
                           'span[@class="text"]/span/text()').extract())

        if eventName:
            teams = eventName.lower().split(' - ')
            l.add_value('teams', teams)

        # MO market first
        # Get table associated with the td that has a p with text "Match"
        # (remember xpath .. means move up just like linux fs)
        MOdict = {'marketName': 'Match Odds'}
        MOtable = response.xpath('//tr/td[@class="bets"]/p[@class="info"]'
                                 '[text()="Match"]/../table')
        home_price = take_first(
            MOtable.xpath('tr/td[1]/p/strong/text()').extract())
        draw_price = take_first(
            MOtable.xpath('tr/td[2]/p/strong/text()').extract())
        away_price = take_first(
            MOtable.xpath('tr/td[3]/p/strong/text()').extract())
        MOdict['runners'] = [
            {
                'runnerName': 'HOME',
                'price': home_price
            },
            {
                'runnerName': 'DRAW',
                'price': draw_price
            },
            {
                'runnerName': 'AWAY',
                'price': away_price
            },
        ]

        # CS market
        CSdict = {'marketName': 'Correct Score', 'runners': []}
        CStable = response.xpath('//tr/td[@class="bets"]/p[@class="info"]'
                                 '[text()="Correct Score"]/../table')
        CSresults = CStable.xpath('tr/td/p')
        for result in CSresults:
            # Deal with ':' in name in loaders
            runnerName = take_first(result.xpath('span/text()').extract())
            price = take_first(result.xpath('strong/text()').extract())
            CSdict['runners'].append({
                'runnerName': runnerName,
                'price': price,
                'reverse_tag': False
            })

        # Add markets
        l.add_value('markets', [MOdict, CSdict])

        # Load item
        return l.load_item()
Exemplo n.º 24
0
    def parse_events(self, response):
        '''
        Loop over all events. If no moreLink
        just yield the match odds data, if moreLink
        pass the MO data in meta and then request other markets
        data, finally adding all mkts in parseData
        '''

        pageNum = response.meta['page']
        log.msg('Parsing events for page: %s' % pageNum)

        try:
            jResp = json.loads(response.body)
            sel = Selector(text=jResp['selections'])
        except ValueError as e:
            log.msg('Error could not load JSON when paging? %s' % e,
                    level=log.ERROR)

        base_url = 'https://www.nordicbet.com/eng/sportsbook'
        headers = {
            'Host': 'www.nordicbet.com',
            'Referer': 'https://www.nordicbet.com/eng/sportsbook',
            'X-Requested-With': 'XMLHttpRequest'
        }
        events = sel.xpath('//table[starts-with(@class, "resultstbl 1x2")]')
        for event in events:
            # Does event have moreLink?
            moreLink = take_first(
                event.xpath('tr/td/div[@class="extra"]/a').extract())
            # eventId needed for call for other markets (only if moreLink)
            eventClass = take_first(event.xpath('@class').extract())
            eId = eventClass.rsplit('_')[-1]

            # Build MO market and other event details
            # Get MO prices only and add to items ready to passed in meta
            # for insertion in parseData (first iteration of loop only)
            teams = []
            eventName = event.xpath(
                './/tr/td/div[@class="label"]/text()'
                '| .//tr/td/div[@class="label"]/a/text()').extract()
            eventName = ''.join([eN.strip() for eN in eventName])
            if eventName:
                teams = eventName.lower().split(' - ')

            dateTime = take_first(
                event.xpath('.//tr/td//span[@class="day"]/text()').extract())
            if not dateTime:
                # Could be live
                livelogo = event.xpath(
                    './/tr/td[2]/div[@class="endingtime"]/div[@class="livebet_now_logo"]'
                )
                if livelogo:
                    dateTime = 'Today'
                else:
                    log.msg('Error no dateTime could be grabbed')

            home_price = take_first(
                event.xpath(
                    './/tr/td[4]//div[@class="odds_middle"]/text()').extract())
            draw_price = take_first(
                event.xpath(
                    './/tr/td[5]//div[@class="odds_middle"]/text()').extract())
            away_price = take_first(
                event.xpath(
                    './/tr/td[6]//div[@class="odds_middle"]/text()').extract())
            mDict = {
                'marketName':
                'Match Odds',
                'runners': [{
                    'runnerName': 'HOME',
                    'price': home_price
                }, {
                    'runnerName': 'DRAW',
                    'price': draw_price
                }, {
                    'runnerName': 'AWAY',
                    'price': away_price
                }]
            }
            if moreLink:
                # Pass along MOmarket and other event details, and make
                # request for more markets
                stamp = int(time.time() * 1000)
                meta = {
                    'teams': teams,
                    'eventName': eventName,
                    'dateTime': dateTime,
                    'mDict': mDict
                }
                # Make request for event detail
                GETstr = ('?cmd=chooseInAjax&sidebet_type=1x2&show_extra=true&'
                          'game_id=%s&source=15&_=%s' % (eId, stamp))

                yield Request(url=base_url + GETstr,
                              headers=headers,
                              meta=meta,
                              dont_filter=True,
                              callback=self.parseData)
            else:
                # Just load the MO market only event
                l = EventLoader(item=EventItem2(), response=response)
                l.add_value('sport', u'Football')
                l.add_value('bookie', self.name)
                l.add_value('dateTime', dateTime)
                l.add_value('teams', teams)
                l.add_value('markets', [
                    mDict,
                ])
                # Debug
                log.msg('No moreLink for %s, load MO prices only' % eventName,
                        level=log.DEBUG)
                # log.msg('Datetime: %s' % dateTime, level=log.DEBUG)
                # log.msg('Teams %s' % teams, level=log.DEBUG)
                # log.msg('mDict: %s' % mDict, level=log.DEBUG)
                # Return loaded item
                # stop = raw_input('e2c')
                yield l.load_item()
Exemplo n.º 25
0
    def parseData(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = response.meta['dateTime']
        l.add_value('dateTime', dateTime)

        teams = response.meta['teams']
        l.add_value('teams', teams)

        # Markets
        # There is not enough regularity for one-size-fits-all but could we
        # at least grab markets in groups sharing similar formatting (CS and
        # HT/FT for example)
        mktBlocks = response.xpath(
            '//table[starts-with(@class, "resultstbl")]')
        allmktdicts = [
            response.meta['mDict'],
        ]  # Add in MO from meta
        underover_re = re.compile("Under/over: (?P<val>\d+\.\d+)")
        for mktBlock in mktBlocks:
            blockName = take_first(
                mktBlock.xpath('tr[1]/td[1]/div/text()').extract())

            if 'Under/over' in blockName:
                # Things are tricky because 'Under/over' or 'Under/over team'
                # is the grouping, but the marketName should actually be the row
                # label, i.e. 'Under/over 0.5', with runners 'Under 0.5', 'Over
                # 0.5'
                rows = mktBlock.xpath('tr[re:test(@class, "item$")]')
                for row in rows:
                    marketName = take_first(
                        row.xpath('td[@class="label"]/div/text()').extract())
                    # If Under/over teams
                    if 'team' in blockName:
                        team = take_first(
                            row.xpath('td[@align="left"]/text()').extract())
                        marketName = team + marketName
                    mDict = {'marketName': marketName, 'runners': []}
                    # Get the numeric over/under value
                    r = underover_re.search(marketName)
                    underover_val = r.groupdict()['val']
                    # Prices
                    uprice = take_first(
                        row.xpath(
                            'td[@class="bet"]//div[@class="odds_middle"]/text()'
                        ).extract())
                    oprice = take_first(
                        row.xpath(
                            'td[@class="bet last"]//div[@class="odds_middle"]/text()'
                        ).extract())
                    mDict['runners'].extend([{
                        'runnerName': 'Under %s' % underover_val,
                        'price': str(uprice),
                    }, {
                        'runnerName': 'Over %s' % underover_val,
                        'price': str(oprice)
                    }])
                    allmktdicts.append(mDict)

                if 'Exact score' == blockName or 'Halftime/fulltime' == blockName:
                    # N.B. Even half-time exact score different format!
                    if 'Exact score' == blockName:
                        marketName = 'Correct Score'
                    elif 'Halftime/fulltime' == blockName:
                        marketName = 'Halftime/Fulltime'
                    mDict = {'marketName': marketName, 'runners': []}
                    runners = mktBlock.xpath('tr[re:test(@class, "item$")]/'
                                             'td[@class="exact_result"]/'
                                             'table/tr/td[@align="center"]/'
                                             'table')
                    for runner in runners:
                        runnerName = take_first(
                            runner.xpath('td[1]/text()').extract())
                        price = take_first(
                            runner.xpath('td[2]//div[@class="odds_middle"]/'
                                         'text()').extract())
                        mDict['runners'].append({
                            'runnerName': runnerName,
                            'price': price
                        })

                    allmktdicts.append(mDict)

        # # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()
Exemplo n.º 26
0
    def parse_Data(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        eventSelection = response.xpath(
            '//table[contains(@class, "event-type")]/tbody/tr')

        items = []
        for event in eventSelection:

            l = EventLoader(item=EventItem2(), response=response)
            l.add_value('sport', u'Football')
            l.add_value('bookie', self.name)

            dateTime = take_first(event.xpath('td[2]/text()').extract())
            # Sometimes just a time 13:30 if date is Today, the following -> ''
            if dateTime:
                dateTime = dateTime.strip()[:-5]
            else:
                dateTime = 'Today'
            if not dateTime or dateTime == ' ':
                dateTime = 'Today'
            l.add_value('dateTime', dateTime)

            eventName = take_first(
                event.xpath('td[@class="event"]/a/span/text()').extract())
            if eventName:
                teams = eventName.lower().split(' v ')
                l.add_value('teams', teams)

            # MO prices
            MOdict = {'marketName': 'Match Odds'}
            home_price = take_first(
                event.xpath('td[5][@class="price"]/a/strong/text()').extract())
            draw_price = take_first(
                event.xpath('td[6][@class="price"]/a/strong/text()').extract())
            away_price = take_first(
                event.xpath('td[7][@class="price"]/a/strong/text()').extract())
            MOdict['runners'] = [
                {
                    'runnerName': 'HOME',
                    'price': home_price
                },
                {
                    'runnerName': 'DRAW',
                    'price': draw_price
                },
                {
                    'runnerName': 'AWAY',
                    'price': away_price
                },
            ]
            # Add markets
            l.add_value('markets', [
                MOdict,
            ])

            # Load item
            items.append(l.load_item())

        return items
Exemplo n.º 27
0
    def parse_Data(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        # Unlike most scrapes there is no detail page for each event, just a
        # listing of markets. We grab the MO, totals and goals type markets for
        # each event (each triplet), but this event may be repeated again with
        # further, different totals or goals markets, so at the end we have to
        # consolidate markets to single event. Also half time versions are dealt
        # with.
        rows = response.xpath(
            '//tr[@class="TrGameOdd" or @class="TrGameEven"]')
        events = list(chunks(rows, 3))
        eventDics = []
        for evt in events:
            eventDic = {}

            team1 = take_first(evt[0].xpath('td[4]/text()').extract())
            team2 = take_first(evt[1].xpath('td[4]/text()').extract())
            teams = [team1, team2]
            eventDic['teams'] = teams

            dateTime = take_first(evt[0].xpath('td[2]/text()').extract())
            eventDic['dateTime'] = dateTime

            eventDic['markets'] = []
            # This regex matches names like o3.75, u.75, o2 and prices like +110
            # or EV.
            Tregex = re.compile(
                "(?P<name>[ou](\d+)?(.\d+)?)(?P<price>([+-]\d+|EV))")
            # Similar to Tregex but also matches PK as name.
            Gregex = re.compile(
                "(?P<name>([+-](\d+)?(.\d+)?|PK))(?P<price>([+-]\d+|EV))")
            for ind in ['5', '6', '7']:
                # index 5 in MO, 6 is totals, 7 is goals
                # NB for Totals and Goals (i.e. ind 6 or 7) the home_price and
                # away_price are really combinations of name and price.
                home_price = take_first(evt[0].xpath('td[' + ind +
                                                     ']/text()').extract())
                away_price = take_first(evt[1].xpath('td[' + ind +
                                                     ']/text()').extract())
                draw_price = take_first(evt[2].xpath('td[' + ind +
                                                     ']/text()').extract())
                if ind == '5' and home_price and away_price and draw_price:
                    marketName = 'Match Odds'
                elif ind == '6' and home_price and away_price:
                    marketName = 'Totals'
                elif ind == '7' and home_price and away_price:
                    marketName = 'Goals'
                else:
                    continue

                # Make sure half time markets have this in marketName, then
                # remove it from team names themselves
                if '1ST HALF' in team1 or ' 1ST HALF' in team2:
                    marketName += ' 1ST HALF'
                    eventDic['teams'] = [
                        n.replace(' 1ST HALF', '') for n in teams
                    ]

                if 'Match Odds' in marketName:
                    mdict = {'marketName': marketName, 'runners': []}
                    mdict['runners'] = [
                        {
                            'runnerName': 'HOME',
                            'price': am2dec(home_price)
                        },
                        {
                            'runnerName': 'DRAW',
                            'price': am2dec(draw_price)
                        },
                        {
                            'runnerName': 'AWAY',
                            'price': am2dec(away_price)
                        },
                    ]

                    eventDic['markets'].append(mdict)
                else:
                    home_price = home_price.encode('utf8').replace('\xc2\xbd', '.5') \
                                                          .replace('\xc2\xbc', '.25') \
                                                          .replace('\xc2\xbe', '.75')
                    away_price = away_price.encode('utf8').replace('\xc2\xbd', '.5') \
                                                          .replace('\xc2\xbc', '.25') \
                                                          .replace('\xc2\xbe', '.75')
                    regex = None
                    if ind == '6':
                        # Totals
                        regex = Tregex
                    elif ind == '7':
                        # Goals
                        regex == Gregex
                    if regex:
                        home = regex.search(home_price)
                        away = regex.search(away_price)
                        if home and away:
                            home_name = home.groupdict()['name']
                            home_price = am2dec(home.groupdict()['price'])
                            away_name = away.groupdict()['name']
                            away_price = am2dec(away.groupdict()['price'])
                            marketName += ' ' + home_name + '/' + away_name
                            mdict = {'marketName': marketName, 'runners': []}
                            mdict['runners'] = [
                                {
                                    'runnerName': home_name,
                                    'price': home_price
                                },
                                {
                                    'runnerName': away_name,
                                    'price': away_price
                                },
                            ]
                            eventDic['markets'].append(mdict)
                        else:
                            log.msg('No regex match: home: %s, away:%s' %
                                    (home_name, away_name),
                                    level=log.ERROR)
                            log.msg('Teams: %s, %s' % (team1, team2),
                                    level=log.ERROR)

            eventDics.append(eventDic)

            # Finally consolidate all markets with same teams and date.
            newDics = []
            for eD1 in eventDics:
                new_eD = {
                    'teams': eD1['teams'],
                    'dateTime': eD1['dateTime'],
                    'markets': []
                }
                mkts = [
                    e['markets'] for e in eventDics
                    if e['teams'] == eD1['teams']
                    and e['dateTime'] == eD1['dateTime']
                ]
                # flatten
                mkts = [item for sublist in mkts for item in sublist]
                new_eD['markets'] = mkts
                # if not appended already add it
                if ((new_eD['teams'], new_eD['dateTime'])
                        not in [(d['teams'], d['dateTime']) for d in newDics]):
                    newDics.append(new_eD)

        nevents = []
        for nevent in newDics:
            l = EventLoader(item=EventItem2(), response=response)
            l.add_value('sport', u'Football')
            l.add_value('bookie', self.name)
            l.add_value('dateTime', nevent['dateTime'])
            l.add_value('teams', nevent['teams'])
            l.add_value('markets', nevent['markets'])
            nevents.append(l.load_item())

        return nevents
Exemplo n.º 28
0
    def parse_Data(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.context['spiderlocale'] = 'es_ES.utf8'  # Set Spanish locale for dates
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = response.meta['dateTime']
        l.add_value('dateTime', dateTime)

        eventName = take_first(response.xpath('//div[@class="breadcrumbs-content"]/'
                                              'ul/li[@class="last-item"]/a/text()').extract())
        if eventName:
            teams = eventName.lower().split(' v ')
            l.add_value('teams', teams)

        # Markets
        # N.B. even appending ?show_all=Y to url still means unexpanded markets
        # need additional get market requests. Not worth it for now.
        mkts = response.xpath('//div[starts-with(@class, "expander ")]')
        allmktdicts = []
        for mkt in mkts:
            marketName = take_first(mkt.xpath('h6/span/text()').extract())
            mdict = {'marketName': marketName, 'runners': []}
            runners = mkt.xpath('div[@class="expander-content"]/table/tbody/tr/td')
            for runner in runners:
                runnername = take_first(runner.xpath('div/button/span/'
                                                     'span[@class="seln-name" or @class="seln-sort"]/'
                                                     'span/text()').extract())
                price = take_first(runner.xpath('div/button/span/'
                                                'span[@class="price dec"]/'
                                                'text()').extract())
                if runnername and price:
                    mdict['runners'].append({'runnerName': runnername, 'price': price})
            if mdict['marketName'] and mdict['runners']:
                allmktdicts.append(mdict)

        # Do some Sportium specific post processing and formating
        for mkt in allmktdicts:
            if '1-X-2' == mkt['marketName']:
                mkt['marketName'] = 'Match Odds'
                for runner in mkt['runners']:
                    if teams[0] in runner['runnerName'].lower():
                        runner['runnerName'] = 'HOME'
                    elif teams[1] in runner['runnerName'].lower():
                        runner['runnerName'] = 'AWAY'
                    elif 'X' == runner['runnerName']:
                        runner['runnerName'] = 'DRAW'
            elif 'Marcador exacto' == mkt['marketName']:
                mkt['marketName'] = 'Correct Score'
            elif 'Descanso/Final del Partido' == mkt['marketName']:
                mkt['marketName'] = 'HALF-TIME/FULL-TIME'

        # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()
Exemplo n.º 29
0
    def parseEvent(self, response):

        log.msg('Going to parse data for treeId: %s' % response.meta['treeId'],
                level=log.DEBUG)

        # Resp is HTML wrapped in JSON
        jResp = json.loads(response.body)
        htmlResp = jResp['HTML_ROW_WITH_ADDITIONAL_MARKETS']
        # Get selector from this unicode string
        sel = Selector(text=htmlResp)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = take_first(
            sel.xpath('//td[starts-with(@class,"first")]/'
                      'table/tr//td[@class="date"]/text()').extract())
        l.add_value('dateTime', dateTime)

        team1 = take_first(
            sel.xpath('//td[@class="name" or @class="today-name"]'
                      '/span/div[1]/text()').extract())
        team2 = take_first(
            sel.xpath('//td[@class="name" or @class="today-name"]'
                      '/span/div[2]/text()').extract())
        if team1 and team2:
            l.add_value('teams', [team1, team2])

        # Markets
        markets = sel.xpath('//div[@class="block-market-table-wrapper"]'
                            '/div[@class="market-inline-block-table-wrapper"]')
        # MO market first
        # Get table associated with the td that has a p with text "Match"
        # (remember xpath .. means move up just like linux fs)
        MOdict = {'marketName': 'Match Odds'}
        # N.B. you may consider using [contains(text(), "Result")], but this
        # matches other markets.
        MOtable = markets.xpath(
            'div[@class="market-table-name"]/'
            'div[@class="name-field"][text()=" Result "]/../../table')
        home_price = take_first(
            MOtable.xpath('tr[1]/td[2]/span/text()').extract())
        draw_price = take_first(
            MOtable.xpath('tr[2]/td[2]/span/text()').extract())
        away_price = take_first(
            MOtable.xpath('tr[3]/td[2]/span/text()').extract())
        MOdict['runners'] = [
            {
                'runnerName': 'HOME',
                'price': home_price
            },
            {
                'runnerName': 'DRAW',
                'price': draw_price
            },
            {
                'runnerName': 'AWAY',
                'price': away_price
            },
        ]

        # CS market
        CSdict = {'marketName': 'Correct Score', 'runners': []}
        CStable = markets.xpath(
            'div[@class="market-table-name"]/div[@class="name-field"]'
            '[text()=" Correct Score "]/../../table')
        CShomeRes = CStable.xpath(
            'tr/td[1]/div[contains(@class, "coeff-link-3way")]')
        CSdrawRes = CStable.xpath(
            'tr/td[2]/div[contains(@class, "coeff-link-3way")]')
        CSawayRes = CStable.xpath(
            'tr/td[3]/div[contains(@class, "coeff-link-3way")]')

        for result in CShomeRes:
            # Column where home team has winning score (strip brackets in loader)
            runnerName = take_first(
                result.xpath('div[@class="coeff-handicap"]/text()').extract())
            price = take_first(
                result.xpath(
                    'div[@class="coeff-price"]/span/text()').extract())
            CSdict['runners'].append({
                'runnerName': runnerName,
                'price': price,
                'reverse_tag': False
            })

        for result in CSdrawRes:
            # Column for draw
            runnerName = take_first(
                result.xpath('div[@class="coeff-handicap"]/text()').extract())
            price = take_first(
                result.xpath(
                    'div[@class="coeff-price"]/span/text()').extract())
            CSdict['runners'].append({
                'runnerName': runnerName,
                'price': price,
                'reverse_tag': False
            })

        for result in CSawayRes:
            # Column for away (needs reversing)
            runnerName = take_first(
                result.xpath('div[@class="coeff-handicap"]/text()').extract())
            price = take_first(
                result.xpath(
                    'div[@class="coeff-price"]/span/text()').extract())
            CSdict['runners'].append({
                'runnerName': runnerName,
                'price': price,
                'reverse_tag': True
            })
        # Add markets
        l.add_value('markets', [MOdict, CSdict])

        # Load item
        return l.load_item()
Exemplo n.º 30
0
    def parseData(self, response):

        log.msg('Going to parse data for URL: %s' % response.url[20:],
                level=log.INFO)

        l = EventLoader(item=EventItem2(), response=response)
        l.add_value('sport', u'Football')
        l.add_value('bookie', self.name)

        dateTime = take_first(response.xpath('//div[@id="ContentWrapper"]/'
                                             'div[@class="antepostContainer"]/'
                                             'div[@class="antepostMarketContainer"]/'
                                             'div[1][@class="antepostEventContainer"]/'
                                             'h4/span/text()').extract())
        # Sometimes instead of divs we get an entire reponse with same class/ids
        # but using tables/tr/td instead of the divs. We could use element wildcards
        # but I think it might be a bit messy.
        if not dateTime:
            dateTime = take_first(response.xpath('//table[@class="antepostEventContainer"][1]/'
                                                 'tr[1]/td[1]/h4/span/text()').extract())
        dateTime = dateTime.split('-')[1]

        dateTime = dateTime.replace('till', '')
        l.add_value('dateTime', dateTime)

        eventName = take_first(response.xpath('//h3[@class="antepostHeader"]/text()').extract())

        if eventName:
            teams = eventName.lower().split(' vs ')
            l.add_value('teams', teams)

        # Markets
        mkts = response.xpath('//div[@class="antepostEventContainer"]')
        if not mkts:
            mkts = response.xpath('//table[@class="antepostEventContainer"]')
        allmktdicts = []
        for mkt in mkts:
            marketName = take_first(mkt.xpath('h4[@class="antepostEventHeader"]/span/text()').extract())
            if not marketName:
                marketName = take_first(mkt.xpath('tr/td/h4[@class="antepostEventHeader"]/span/text()').extract())
            marketName = take_first(marketName.split(' - '))
            mdict = {'marketName': marketName, 'runners': []}
            if 'Correct Score' in marketName:
                runners = mkt.xpath('div[@class="correctScoreContainer"]/'
                                    'div[not(@class="clear")]/div[starts-with(@class, "selection")]')
                if not runners:
                    runners = mkt.xpath('tr/td/table[@class="correctScoreContainer"]/'
                                        'tr/td/table[not(@class="clear")]/'
                                        'tr/td/table[starts-with(@class, "selection")]/tr/td')
                for runner in runners:
                    runnername = take_first(runner.xpath('span[@class="name"]/text()').extract())
                    price = take_first(runner.xpath('span[@class="price"]/text()').extract())
                    mdict['runners'].append({'runnerName': runnername,
                                             'price': price,
                                             })
            else:
                runners = mkt.xpath('div[@class="antepostSelectionContainer"]/'
                                    'div[starts-with(@class, "antepostSelection")]')
                if not runners:
                    runners = mkt.xpath('tr/td/table[@class="antepostSelectionContainer"]/'
                                        'tr/td/table[starts-with(@class, "antepostSelection")]/tr/td')
                for runner in runners:
                    runnerName = take_first(runner.xpath('span[@class="antepostSelectionName"]/text()').extract())
                    price = take_first(runner.xpath('span[@class="antepostSelectionOdds"]/text()').extract())
                    mdict['runners'].append({'runnerName': runnerName, 'price': price})
            allmktdicts.append(mdict)

        # Do some BGbet specific post processing and formating
        for mkt in allmktdicts:
            if 'Ninety Minutes' == mkt['marketName']:
                mkt['marketName'] = 'Match Odds'
                for runner in mkt['runners']:
                    if teams[0] in runner['runnerName'].lower():
                        runner['runnerName'] = 'HOME'
                    elif teams[1] in runner['runnerName'].lower():
                        runner['runnerName'] = 'AWAY'
                    elif 'Draw' in runner['runnerName']:
                        runner['runnerName'] = 'DRAW'
            elif 'Correct Scores' in mkt['marketName']:
                mkt['marketName'] = 'Correct Score'
                for runner in mkt['runners']:
                    if teams[1] in runner['runnerName'].lower():
                        runner['reverse_tag'] = True
                    else:
                        runner['reverse_tag'] = False

        # Add markets
        l.add_value('markets', allmktdicts)

        # Load item
        return l.load_item()