def parse_Data(self, response): log.msg('Going to parse data for URL: %s' % response.url[20:], level=log.INFO) l = EventLoader(item=EventItem2(), response=response) l.add_value('sport', u'Football') l.add_value('bookie', self.name) dateTime = take_first( response.xpath('//div[@id="center_content"]/' 'div[@class="coupon_header scrollable"]/' 'div[@class="coupon_header_titles"]/' 'h4/span/text()').extract()) l.add_value('dateTime', dateTime) eventName = take_first( response.xpath('//div[@id="center_content"]/' 'div[@class="coupon_header scrollable"]/' 'div[@class="coupon_header_titles"]/' 'h1/@title').extract()) if eventName: teams = eventName.lower().split(' v ') l.add_value('teams', teams) # Markets mkts = response.xpath( '//div[@class="single_markets" or @class="multiple_markets"]/' 'div[starts-with(@id, "coupon")]') allmktdicts = [] for mkt in mkts: marketName = take_first(mkt.xpath('h4/text()').extract()) mdict = {'marketName': marketName, 'runners': []} runners = mkt.xpath( 'table[not(@class="has_group_date")]/' 'tbody/tr[not(@class="header")]/td[@class="outcome_td"]') for runner in runners: runnerName = take_first( runner.xpath('span/@data-outcome_description').extract()) price = take_first( runner.xpath( 'span/a/span[@class="price"]/text()').extract()) mdict['runners'].append({ 'runnerName': runnerName, 'price': price }) allmktdicts.append(mdict) # Do some Betvic specific post processing and formating for mkt in allmktdicts: if 'Match Betting' in mkt['marketName']: mkt['marketName'] = 'Match Odds' for runner in mkt['runners']: if teams[0] in runner['runnerName'].lower(): runner['runnerName'] = 'HOME' elif teams[1] in runner['runnerName'].lower(): runner['runnerName'] = 'AWAY' elif 'Draw' in runner['runnerName']: runner['runnerName'] = 'DRAW' elif 'Correct Score - 90 Mins' in mkt['marketName']: mkt['marketName'] = 'Correct Score' for runner in mkt['runners']: if teams[1] in runner['runnerName'].lower(): runner['reverse_tag'] = True else: runner['reverse_tag'] = False # Add markets l.add_value('markets', allmktdicts) # Load item return l.load_item()
def parse_Data(self, response): log.msg('Going to parse data for URL: %s' % response.url[20:], level=log.INFO) l = EventLoader(item=EventItem2(), response=response) l.add_value('sport', u'Football') l.add_value('bookie', self.name) all_scripts = response.xpath('//script') wanted_script = take_first([script for script in all_scripts if 'Sportsbook.AppView(' in script.extract()]) jsonEventsData = json.loads(wanted_script.extract().splitlines()[16].lstrip()[4:-1]) eventSelection = take_first(jsonEventsData['events']) marketSelection = take_first(jsonEventsData['markets'].values()) runnerSelection = jsonEventsData['selections'] dateTime = eventSelection['s'] # ms since epoch to s (floor div is fine) dateTime = dateTime/1000 dateTime = pydt.fromtimestamp(dateTime).strftime('%m %d') l.add_value('dateTime', dateTime) eventName = eventSelection['n'] if eventName: teams = eventName.lower().split(' v ') l.add_value('teams', teams) # Markets allmktdicts = [] for mkt in marketSelection: marketName = mkt['n'] marketId = mkt['id'] mdict = {'marketName': marketName, 'runners': []} runners = runnerSelection[str(marketId)] for runner in runners: try: runnername = runner['n'] except KeyError: # player to score markets (I think you would need the names # data, i.e. jsonEventsData['names'], then match on ids again continue if runner['ps']: # If non-empty list price = runner['ps'][0]['v'] else: # empty-list (sometimes ps is just [], not sure why, url # shows prices for affected mkts, usually mkts are totals or # handicapped) price = None # log.msg('Price list was empty for url: %s' % response.url, # level=log.ERROR) # log.msg('Marketname was %s \n with runner dump: %s' % (marketName, runner), # level=log.ERROR) if runnername and price: mdict['runners'].append({'runnerName': runnername, 'price': price}) allmktdicts.append(mdict) # Do some Gentingbet specific post processing and formating for mkt in allmktdicts: if 'Match Result' == mkt['marketName']: mkt['marketName'] = 'Match Odds' for runner in mkt['runners']: if teams[0] in runner['runnerName'].lower(): runner['runnerName'] = 'HOME' elif teams[1] in runner['runnerName'].lower(): runner['runnerName'] = 'AWAY' elif 'Draw' in runner['runnerName']: runner['runnerName'] = 'DRAW' elif 'Correct Score' in mkt['marketName']: for runner in mkt['runners']: if teams[1] in runner['runnerName'].lower(): runner['reverse_tag'] = True else: runner['reverse_tag'] = False # Add markets l.add_value('markets', allmktdicts) # Load item return l.load_item()
def parse_Data(self, response): log.msg('Going to parse data for URL: %s' % response.url, level=log.INFO) l = EventLoader(item=EventItem2(), response=response) l.add_value('sport', u'Football') l.add_value('bookie', self.name) dateTime = response.meta['dateTime'] l.add_value('dateTime', dateTime) eventName = take_first( response.xpath('//h2[@class="AllBetsEventName"]/' 'text()').extract()) if eventName: teams = eventName.lower().split(' v ') l.add_value('teams', teams) mkts = response.xpath('//li[starts-with(@class, "m_item")]') # N.B. just like a filesystem path we go upward with '..', after # we find the ones with Match Result only MOdict = {'marketName': 'Match Odds', 'runners': []} MOrunners = mkts.xpath('./span[starts-with(@class, "headerSub")]/' 'span[text()="Match Prices"]/../../' 'ul') # MO prices home_price = take_first( MOrunners.xpath( './/div[@class="odds home"]/' 'div[@id="isOffered"]/' 'a/span[@class="priceText wide UK"]/text()').extract()) draw_price = take_first( MOrunners.xpath( './/div[@class="odds draw"]/' 'div[@id="isOffered"]/' 'a/span[@class="priceText wide UK"]/text()').extract()) away_price = take_first( MOrunners.xpath( './/div[@class="odds away"]/' 'div[@id="isOffered"]/' 'a/span[@class="priceText wide UK"]/text()').extract()) MOdict['runners'] = [ { 'runnerName': 'HOME', 'price': home_price }, { 'runnerName': 'DRAW', 'price': draw_price }, { 'runnerName': 'AWAY', 'price': away_price }, ] # Correct Score CSdict = {'marketName': 'Correct Score', 'runners': []} CSresults = mkts.xpath('./span[starts-with(@class, "headerSub")]/' 'span[text()="Correct Score"]/../../' 'ul//div[@class="m_event"]') CSdict = {'marketName': 'Correct Score', 'runners': []} for result in CSresults: # type i.e. home draw or away (used to decide whether needs reversing) CStype = take_first( result.xpath('div[@class="results"]/' 'div[starts-with(@id, "s_")]/@class').extract()) runnerName = take_first( result.xpath('div[@class="description"]/@title').extract()) price = take_first( result.xpath( 'div[@class="results"]/' 'div[starts-with(@id, "s_")]/' 'div[@id="isOffered"]/a/' 'span[@class="priceText wide UK"]/text()').extract()) if runnerName and price: if 'away' in CStype: # Tag for score reversing by loader (e.g. if Team2 1-0 want # just 0-1 to match Betfair format and avoid team name comp) CSdict['runners'].append({ 'runnerName': runnerName, 'price': price, 'reverse_tag': True }) else: CSdict['runners'].append({ 'runnerName': runnerName, 'price': price, 'reverse_tag': False }) # Add markets l.add_value('markets', [MOdict, CSdict]) # Load item return l.load_item()
def parseData(self, response): log.msg('Going to parse data for URL: %s' % response.url[20:], level=log.INFO) l = EventLoader(item=EventItem2(), response=response) l.add_value('sport', u'Football') l.add_value('bookie', self.name) dateTime = take_first(response.xpath('//span[@class="h-fontWeightNormal h-fontSize-11-lineheight-13"]/' 'text()').extract()) dateTime = take_first(dateTime.split(',')) l.add_value('dateTime', dateTime) eventName = take_first(response.xpath('//span[@class="s-selected h-fontSize-14-lineheight-18"]/' 'text()').extract()) if eventName: teams = eventName.lower().split(' - ') l.add_value('teams', teams) # Markets mkts = response.xpath('//div[@class="rB S22G h-bG-FFFFFF l-mb3 l-overflowHidden"]') allmktdicts = [] regex = re.compile("betslip3.OnTipClick\((?P<jData>.+)\)") for mkt in mkts: marketName = take_first(mkt.xpath('table/thead/tr/' 'td[starts-with(@class, "ods-header")]//text()').extract()) if not marketName: log.msg('No marketName, extract from: \n %s' % mkt.xpath('table/thead').extract()) stop = raw_input('e2c') if 'Under/Over' in marketName or 'Handicap' in marketName: # Get handicap type htype = take_first(mkt.xpath('table/tbody/tr/td[1]/text()').extract()) # Extract contents of (..) b1 = htype.find('(') b2 = htype.find(')') marketName += ' ' + htype[b1+1: b2] mDict = {'marketName': marketName, 'runners': []} runners = mkt.xpath('table/tbody/tr/td[starts-with(@class' ', "ods-tbody-td")]/a/@onclick').extract() for runner in runners: r = regex.search(runner) dataDic = r.groupdict()['jData'] dataJSON = json.loads(dataDic) runnerName = dataJSON['TipName'] price = dataJSON['Odd'] mDict['runners'].append({'runnerName': runnerName, 'price': str(price), }) allmktdicts.append(mDict) # Some Betathome specific formatting for mkt in allmktdicts: if mkt['marketName'] == 'Tip': mkt['marketName'] = 'Match Odds' for runner in mkt['runners']: if runner['runnerName'] == u'1': runner['runnerName'] = 'HOME' elif runner['runnerName'] == u'2': runner['runnerName'] = 'AWAY' elif runner['runnerName'] == u'X': runner['runnerName'] = 'DRAW' # Add markets l.add_value('markets', allmktdicts) # Load item # If MOlist (MO only event items) in list load it too # This list should only be present once! try: MOlist = response.meta['MOlist'] log.msg('MOlist recieved appended item to it') MOlist.append(l.load_item()) # in-place change return MOlist except KeyError: pass # Return just the detailed item return l.load_item()