def parse_Data(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(
response.xpath('//div[@id="center_content"]/'
'div[@class="coupon_header scrollable"]/'
'div[@class="coupon_header_titles"]/'
'h4/span/text()').extract())
l.add_value('dateTime', dateTime)
eventName = take_first(
response.xpath('//div[@id="center_content"]/'
'div[@class="coupon_header scrollable"]/'
'div[@class="coupon_header_titles"]/'
'h1/@title').extract())
if eventName:
teams = eventName.lower().split(' v ')
l.add_value('teams', teams)
# Markets
mkts = response.xpath(
'//div[@class="single_markets" or @class="multiple_markets"]/'
'div[starts-with(@id, "coupon")]')
allmktdicts = []
for mkt in mkts:
marketName = take_first(mkt.xpath('h4/text()').extract())
mdict = {'marketName': marketName, 'runners': []}
runners = mkt.xpath(
'table[not(@class="has_group_date")]/'
'tbody/tr[not(@class="header")]/td[@class="outcome_td"]')
for runner in runners:
runnerName = take_first(
runner.xpath('span/@data-outcome_description').extract())
price = take_first(
runner.xpath(
'span/a/span[@class="price"]/text()').extract())
mdict['runners'].append({
'runnerName': runnerName,
'price': price
})
allmktdicts.append(mdict)
# Do some Betvic specific post processing and formating
for mkt in allmktdicts:
if 'Match Betting' in mkt['marketName']:
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if teams[0] in runner['runnerName'].lower():
runner['runnerName'] = 'HOME'
elif teams[1] in runner['runnerName'].lower():
runner['runnerName'] = 'AWAY'
elif 'Draw' in runner['runnerName']:
runner['runnerName'] = 'DRAW'
elif 'Correct Score - 90 Mins' in mkt['marketName']:
mkt['marketName'] = 'Correct Score'
for runner in mkt['runners']:
if teams[1] in runner['runnerName'].lower():
runner['reverse_tag'] = True
else:
runner['reverse_tag'] = False
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parse_Data(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
all_scripts = response.xpath('//script')
wanted_script = take_first([script for script in all_scripts
if 'Sportsbook.AppView(' in script.extract()])
jsonEventsData = json.loads(wanted_script.extract().splitlines()[16].lstrip()[4:-1])
eventSelection = take_first(jsonEventsData['events'])
marketSelection = take_first(jsonEventsData['markets'].values())
runnerSelection = jsonEventsData['selections']
dateTime = eventSelection['s']
# ms since epoch to s (floor div is fine)
dateTime = dateTime/1000
dateTime = pydt.fromtimestamp(dateTime).strftime('%m %d')
l.add_value('dateTime', dateTime)
eventName = eventSelection['n']
if eventName:
teams = eventName.lower().split(' v ')
l.add_value('teams', teams)
# Markets
allmktdicts = []
for mkt in marketSelection:
marketName = mkt['n']
marketId = mkt['id']
mdict = {'marketName': marketName, 'runners': []}
runners = runnerSelection[str(marketId)]
for runner in runners:
try:
runnername = runner['n']
except KeyError:
# player to score markets (I think you would need the names
# data, i.e. jsonEventsData['names'], then match on ids again
continue
if runner['ps']:
# If non-empty list
price = runner['ps'][0]['v']
else:
# empty-list (sometimes ps is just [], not sure why, url
# shows prices for affected mkts, usually mkts are totals or
# handicapped)
price = None
# log.msg('Price list was empty for url: %s' % response.url,
# level=log.ERROR)
# log.msg('Marketname was %s \n with runner dump: %s' % (marketName, runner),
# level=log.ERROR)
if runnername and price:
mdict['runners'].append({'runnerName': runnername, 'price': price})
allmktdicts.append(mdict)
# Do some Gentingbet specific post processing and formating
for mkt in allmktdicts:
if 'Match Result' == mkt['marketName']:
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if teams[0] in runner['runnerName'].lower():
runner['runnerName'] = 'HOME'
elif teams[1] in runner['runnerName'].lower():
runner['runnerName'] = 'AWAY'
elif 'Draw' in runner['runnerName']:
runner['runnerName'] = 'DRAW'
elif 'Correct Score' in mkt['marketName']:
for runner in mkt['runners']:
if teams[1] in runner['runnerName'].lower():
runner['reverse_tag'] = True
else:
runner['reverse_tag'] = False
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parse_Data(self, response):
log.msg('Going to parse data for URL: %s' % response.url,
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = response.meta['dateTime']
l.add_value('dateTime', dateTime)
eventName = take_first(
response.xpath('//h2[@class="AllBetsEventName"]/'
'text()').extract())
if eventName:
teams = eventName.lower().split(' v ')
l.add_value('teams', teams)
mkts = response.xpath('//li[starts-with(@class, "m_item")]')
# N.B. just like a filesystem path we go upward with '..', after
# we find the ones with Match Result only
MOdict = {'marketName': 'Match Odds', 'runners': []}
MOrunners = mkts.xpath('./span[starts-with(@class, "headerSub")]/'
'span[text()="Match Prices"]/../../'
'ul')
# MO prices
home_price = take_first(
MOrunners.xpath(
'.//div[@class="odds home"]/'
'div[@id="isOffered"]/'
'a/span[@class="priceText wide UK"]/text()').extract())
draw_price = take_first(
MOrunners.xpath(
'.//div[@class="odds draw"]/'
'div[@id="isOffered"]/'
'a/span[@class="priceText wide UK"]/text()').extract())
away_price = take_first(
MOrunners.xpath(
'.//div[@class="odds away"]/'
'div[@id="isOffered"]/'
'a/span[@class="priceText wide UK"]/text()').extract())
MOdict['runners'] = [
{
'runnerName': 'HOME',
'price': home_price
},
{
'runnerName': 'DRAW',
'price': draw_price
},
{
'runnerName': 'AWAY',
'price': away_price
},
]
# Correct Score
CSdict = {'marketName': 'Correct Score', 'runners': []}
CSresults = mkts.xpath('./span[starts-with(@class, "headerSub")]/'
'span[text()="Correct Score"]/../../'
'ul//div[@class="m_event"]')
CSdict = {'marketName': 'Correct Score', 'runners': []}
for result in CSresults:
# type i.e. home draw or away (used to decide whether needs reversing)
CStype = take_first(
result.xpath('div[@class="results"]/'
'div[starts-with(@id, "s_")]/@class').extract())
runnerName = take_first(
result.xpath('div[@class="description"]/@title').extract())
price = take_first(
result.xpath(
'div[@class="results"]/'
'div[starts-with(@id, "s_")]/'
'div[@id="isOffered"]/a/'
'span[@class="priceText wide UK"]/text()').extract())
if runnerName and price:
if 'away' in CStype:
# Tag for score reversing by loader (e.g. if Team2 1-0 want
# just 0-1 to match Betfair format and avoid team name comp)
CSdict['runners'].append({
'runnerName': runnerName,
'price': price,
'reverse_tag': True
})
else:
CSdict['runners'].append({
'runnerName': runnerName,
'price': price,
'reverse_tag': False
})
# Add markets
l.add_value('markets', [MOdict, CSdict])
# Load item
return l.load_item()
def parseData(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(response.xpath('//span[@class="h-fontWeightNormal h-fontSize-11-lineheight-13"]/'
'text()').extract())
dateTime = take_first(dateTime.split(','))
l.add_value('dateTime', dateTime)
eventName = take_first(response.xpath('//span[@class="s-selected h-fontSize-14-lineheight-18"]/'
'text()').extract())
if eventName:
teams = eventName.lower().split(' - ')
l.add_value('teams', teams)
# Markets
mkts = response.xpath('//div[@class="rB S22G h-bG-FFFFFF l-mb3 l-overflowHidden"]')
allmktdicts = []
regex = re.compile("betslip3.OnTipClick\((?P<jData>.+)\)")
for mkt in mkts:
marketName = take_first(mkt.xpath('table/thead/tr/'
'td[starts-with(@class, "ods-header")]//text()').extract())
if not marketName:
log.msg('No marketName, extract from: \n %s' % mkt.xpath('table/thead').extract())
stop = raw_input('e2c')
if 'Under/Over' in marketName or 'Handicap' in marketName:
# Get handicap type
htype = take_first(mkt.xpath('table/tbody/tr/td[1]/text()').extract())
# Extract contents of (..)
b1 = htype.find('(')
b2 = htype.find(')')
marketName += ' ' + htype[b1+1: b2]
mDict = {'marketName': marketName, 'runners': []}
runners = mkt.xpath('table/tbody/tr/td[starts-with(@class'
', "ods-tbody-td")]/a/@onclick').extract()
for runner in runners:
r = regex.search(runner)
dataDic = r.groupdict()['jData']
dataJSON = json.loads(dataDic)
runnerName = dataJSON['TipName']
price = dataJSON['Odd']
mDict['runners'].append({'runnerName': runnerName,
'price': str(price),
})
allmktdicts.append(mDict)
# Some Betathome specific formatting
for mkt in allmktdicts:
if mkt['marketName'] == 'Tip':
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if runner['runnerName'] == u'1':
runner['runnerName'] = 'HOME'
elif runner['runnerName'] == u'2':
runner['runnerName'] = 'AWAY'
elif runner['runnerName'] == u'X':
runner['runnerName'] = 'DRAW'
# Add markets
l.add_value('markets', allmktdicts)
# Load item
# If MOlist (MO only event items) in list load it too
# This list should only be present once!
try:
MOlist = response.meta['MOlist']
log.msg('MOlist recieved appended item to it')
MOlist.append(l.load_item()) # in-place change
return MOlist
except KeyError:
pass
# Return just the detailed item
return l.load_item()