def parse_Data(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(
response.xpath('//div[@id="event-info"]/'
'span[not(@class="more")]/text()').extract())
dateTime = take_first(dateTime.split(','))
l.add_value('dateTime', dateTime)
eventName = take_first(
response.xpath('//div[@id="event-info"]/'
'h1[@class="league-name"]/text()').extract())
if eventName:
teams = eventName.lower().split(' - ')
l.add_value('teams', teams)
# Markets
mkts = response.xpath('//div[starts-with(@id, "game-wrapper-s-")]')
allmktdicts = []
for mkt in mkts:
marketName = take_first(mkt.xpath('div/h6/text()').extract())
mdict = {'marketName': marketName, 'runners': []}
runners = mkt.xpath('div/ul/li/table/tr/td/form/button')
for runner in runners:
runnername = take_first(
runner.xpath(
'span[@class="option-name"]/text()').extract())
price = take_first(
runner.xpath('span[@class="odds"]/text()').extract())
mdict['runners'].append({
'runnerName': runnername,
'price': price
})
allmktdicts.append(mdict)
# Do some Bwin specific post processing and formating
for mkt in allmktdicts:
if '3-way - result' == mkt['marketName']:
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if teams[0].strip() in runner['runnerName'].lower():
runner['runnerName'] = 'HOME'
elif teams[1].strip() in runner['runnerName'].lower():
runner['runnerName'] = 'AWAY'
elif 'X' == runner['runnerName']:
runner['runnerName'] = 'DRAW'
elif 'Goal bet (regular time)' == mkt['marketName']:
mkt['marketName'] = 'Correct Score'
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parse_Data(self, response):
log.msg('Going to parse data for URL: %s' % response.url, level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = response.meta['dateTime']
l.add_value('dateTime', dateTime)
eventName = take_first(response.xpath('//div[@class="market_title_top"]/'
'h1/text()').extract())
if eventName:
teams = eventName.lower().split(' vs ')
l.add_value('teams', teams)
# Markets
mkts = response.xpath('//div[@class="market_wrapper"]')
allmktdicts = []
for mkt in mkts:
marketName = take_first(mkt.xpath('div[@class="market_title"]/h2/text()').extract())
mdict = {'marketName': marketName, 'runners': []}
runners = mkt.xpath('div[@class="market_content_wrapper"]/'
'div[@class="market_content"]/table/'
'tbody/tr/td[@class="odds"]')
for runner in runners:
runnerName = take_first(runner.xpath('a/@title').extract())
price = take_first(runner.xpath('a/text()').extract())
if runnerName and price:
mdict['runners'].append({'runnerName': runnerName, 'price': price})
allmktdicts.append(mdict)
# Do some Totesport specific post processing and formating
for mkt in allmktdicts:
if '90 Minutes' == mkt['marketName']:
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if teams[0] in runner['runnerName'].lower():
runner['runnerName'] = 'HOME'
elif teams[1] in runner['runnerName'].lower():
runner['runnerName'] = 'AWAY'
elif 'Draw' in runner['runnerName']:
runner['runnerName'] = 'DRAW'
elif 'Correct Score' == mkt['marketName']:
for runner in mkt['runners']:
try:
if teams[1] in runner['runnerName'].lower():
runner['reverse_tag'] = True
else:
runner['reverse_tag'] = False
except AttributeError:
print runner
continue
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parseData(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
# JSON is between _jqjsp(....); so extract with regex
matches = re.findall(r'_jqjsp\((.+?)\);', response.body)
if matches:
jsonResp = json.loads(matches[0])
try:
jsonData = jsonResp['FetchPrematchEventMarketsByEventIdJSONPResult']
except KeyError as e:
log.msg('KeyError: %s' % (self.name, e), level=log.ERROR)
log.msg('Error response or 404?' % (self.name), level=log.ERROR)
log.msg('response dump: %s ' % (self.name, jsonResp), level=log.ERROR)
return []
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = jsonData['DeadlineUTC']
l.add_value('dateTime', dateTime)
eventName = jsonData['Name']
if eventName:
teams = eventName.lower().split(' - ')
l.add_value('teams', teams)
# Markets
mkts = jsonData['Markets']
allmktdicts = []
for mkt in mkts:
marketName = mkt['Name']
mdict = {'marketName': marketName, 'runners': []}
runners = mkt['Selections']
for runner in runners:
runnername = runner['Name']
price = runner['Odds']
mdict['runners'].append({'runnerName': runnername, 'price': price})
allmktdicts.append(mdict)
# Do some Betsafe specific post processing and formating
for mkt in allmktdicts:
if mkt['marketName'] == 'Match Winner':
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if runner['runnerName'] == u'1':
runner['runnerName'] = 'HOME'
elif runner['runnerName'] == u'2':
runner['runnerName'] = 'AWAY'
elif runner['runnerName'] == u'X':
runner['runnerName'] = 'DRAW'
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parseData(self, response):
jResp = json.loads(response.body)
try:
jsonEventsData = take_first(jResp['events'])
jsonOddsData = jResp['betoffers']
except KeyError as e:
log.msg(e, level=log.ERROR)
log.msg('Error response or 404? ', level=log.ERROR)
log.msg('Response dump: \n %s' % jResp)
return []
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = jsonEventsData['start']
l.add_value('dateTime', dateTime)
teams = [jsonEventsData['homeName'], jsonEventsData['awayName']]
l.add_value('teams', teams)
# Markets
allmktdicts = []
for mkt in jsonOddsData:
marketName = mkt['criterion']['label']
mdict = {'marketName': marketName, 'runners': []}
if 'outcomes' in mkt.keys():
runners = mkt['outcomes']
for runner in runners:
runnerName = runner['label']
if 'line' in runner.keys():
# For some reason the over/under markets like 10.5
# formatted as 1050 in line field.
runnerName += ' %.1f' % (runner['line']/1000)
price = runner['oddsFractional']
mdict['runners'].append({'runnerName': runnerName, 'price': price})
allmktdicts.append(mdict)
# Do some 888 specific post processing and formating
for mkt in allmktdicts:
if 'Full Time' == mkt['marketName']:
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if runner['runnerName'] == u'1':
runner['runnerName'] = 'HOME'
elif runner['runnerName'] == u'2':
runner['runnerName'] = 'AWAY'
elif runner['runnerName'] == u'X':
runner['runnerName'] = 'DRAW'
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parse_Data(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = response.meta['dateTime']
l.add_value('dateTime', dateTime)
eventName = response.meta['eventName']
if eventName:
teams = eventName.lower().split('-')
l.add_value('teams', teams)
# Markets
MOdict = response.meta['MOdict']
allmktdicts = [
MOdict,
]
tables = response.xpath('//table[starts-with(@id, "betTable--")]')
blockTitle = ''
block_col_heads = []
for table in tables:
title = take_first(
table.xpath('thead/tr/th[@class="col_title_info"]/'
'span/text()').extract())
col_heads = table.xpath(
'thead/tr/th[@class="col_bet"]/span/text()').extract()
if title:
# Continue to use this title, and update title for block
blockTitle = title
else:
# Use title for block
title = blockTitle
if col_heads:
block_col_heads = col_heads
else:
col_heads = block_col_heads
marketName = take_first(
table.xpath('tbody/tr/td[@class="col_title"]/'
'div/span/a/text()').extract())
prices = table.xpath(
'tbody/tr/td[starts-with(@class, "col_bet")]/a/'
'@data-rate').extract()
mDict = {'marketName': title + ' ' + marketName, 'runners': []}
headPrices = zip(col_heads, prices)
for hP in headPrices:
mDict['runners'].append({'runnerName': hP[0], 'price': hP[1]})
allmktdicts.append(mDict)
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parse_Data(self, response):
league_name = response.meta['league_name']
BpEventId = response.meta['BpEventId']
log.msg('Parsing data for url %s' % response.url)
log.msg('Parsing for league %s and event Id %s' % (league_name, BpEventId))
jResp = json.loads(response.body)
try:
jData = jResp['response']
except (TypeError, KeyError):
jData = jResp['error']
log.msg('Error response for %s and event Id %s' % (league_name, BpEventId),
level=log.ERROR)
log.msg('Error response: %s' % jData, level=log.ERROR)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
try:
dateTime = jData['StartTimeUTC']
l.add_value('dateTime', dateTime)
except (TypeError, KeyError):
log.msg('KeyError jData dump is: %s\n KeyError for %s and event Id %s'
% (jData, league_name, BpEventId), level=log.ERROR)
return []
eventName = jData['name']
if eventName:
teams = eventName.lower().split(' v ')
l.add_value('teams', teams)
# More ambitiously, grab all markets
# First all unique market names
mktNames = set()
for seln in jData['Selections']:
mktNames.add(seln['MarketName'])
# Now construct a market dic for each unique market name
allMktDicts = []
for mktName in mktNames:
mDict = {'marketName': mktName, 'runners': []}
for seln in jData['Selections']:
if seln['MarketName'] == mktName:
runnerName = seln['BettingAlternativeName'].upper()
price = seln['Odds']
stake = seln['Stake']
mDict['runners'].append({'runnerName': runnerName,
'price': price,
'stake': stake,
})
allMktDicts.append(mDict)
# Add markets
l.add_value('markets', allMktDicts)
# Load item
return l.load_item()
def parse_Data(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
eventSelection = response.xpath('//markets//market')
items = []
for event in eventSelection:
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(event.xpath('tsstart/text()').extract())
l.add_value('dateTime', dateTime)
eventName = take_first(event.xpath('eventname/text()').extract())
if eventName:
teams = eventName.lower().split(' v ')
l.add_value('teams', teams)
MOdict = {'marketName': 'Match Odds', 'runners': []}
runners = event.xpath('selections//selection')
for runner in runners:
runType = runner.xpath('hadvalue/text()').extract()
odd_up = runner.xpath('currentpriceup/text()').extract()
odd_down = runner.xpath('currentpricedown/text()').extract()
if odd_up and odd_down:
if runType == [u'H']:
MOdict['runners'].append({
'runnerName':
'HOME',
'price':
odd_up[0] + '/' + odd_down[0]
})
elif runType == [u'D']:
MOdict['runners'].append({
'runnerName':
'DRAW',
'price':
odd_up[0] + '/' + odd_down[0]
})
elif runType == [u'A']:
MOdict['runners'].append({
'runnerName':
'AWAY',
'price':
odd_up[0] + '/' + odd_down[0]
})
l.add_value('markets', [
MOdict,
])
items.append(l.load_item())
return items
def parseData(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(response.xpath('//hgroup/h1/time/@datetime').extract())
l.add_value('dateTime', dateTime)
eventName = response.xpath('//hgroup/h1/text()').extract()
eventName = ''.join([s.strip() for s in eventName])
if eventName:
teams = eventName.lower().split(' - ')
l.add_value('teams', teams)
# Markets
mkts = response.xpath('//div[starts-with(@class, "rollup market_type market_type_id_")]')
allmktdicts = []
for mkt in mkts:
marketName = take_first(mkt.xpath('div[@class="market_type_title"]/h2/'
'text()').extract())
mdict = {'marketName': marketName, 'runners': []}
runners = mkt.xpath('.//tr[@class="event"]/td[starts-with(@class, "outcome outcome_")]')
for runner in runners:
runnername = take_first(runner.xpath('a/div/span[starts-with(@class, "name")]/text()').extract())
price = take_first(runner.xpath('a/@data-price-decimal').extract())
mdict['runners'].append({'runnerName': runnername, 'price': price})
allmktdicts.append(mdict)
# Do some Apostas specific post processing and formating
for mkt in allmktdicts:
if '1X2 - 90 Min' in mkt['marketName']:
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if teams[0] in runner['runnerName'].lower():
runner['runnerName'] = 'HOME'
elif teams[1] in runner['runnerName'].lower():
runner['runnerName'] = 'AWAY'
elif 'empate' in runner['runnerName'].lower():
runner['runnerName'] = 'DRAW'
elif u'Resultado exato' in mkt['marketName']:
mkt['marketName'] = 'Correct Score'
for runner in mkt['runners']:
if teams[1] in runner['runnerName'].lower():
runner['reverse_tag'] = True
else:
runner['reverse_tag'] = False
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parseData(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
jsonBody = json.loads(response.body)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = jsonBody['startTime']
l.add_value('dateTime', dateTime)
eventName = jsonBody['name']
if eventName:
teams = eventName.lower().split(' - ')
if len(teams) != 2:
teams = eventName.lower().split(' v ')
l.add_value('teams', teams)
# Markets
mkts = jsonBody['children']
allmktdicts = []
for mkt in mkts:
marketName = mkt['name']
mdict = {'marketName': marketName, 'runners': []}
runners = mkt['children']
for runner in runners:
runnername = runner['name']
price = runner['odds']
mdict['runners'].append({
'runnerName': runnername,
'price': price
})
allmktdicts.append(mdict)
# Do some Skybet specific post processing and formating
for mkt in allmktdicts:
if mkt['marketName'] == '1X2':
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if teams[0] in runner['runnerName'].lower():
runner['runnerName'] = 'HOME'
elif teams[1] in runner['runnerName'].lower():
runner['runnerName'] = 'AWAY'
elif 'Draw' in runner['runnerName']:
runner['runnerName'] = 'DRAW'
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parse_Data(self, response):
sel = Selector(response)
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
eventSelection = sel.xpath('//markets//market')
log.msg('Number of events in Selection: %s .' % len(eventSelection),
level=log.INFO)
items = []
for event in eventSelection:
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
# '2014-03-24T18:00:00'
dt = event.xpath('tsstart/text()').extract()
l.add_value('dateTime', dt)
eventName = take_first(event.xpath('eventname/text()').extract())
if eventName:
teams = eventName.lower().split(' v ')
l.add_value('teams', teams)
# Markets
MOdict = {'marketName': 'Match Odds'}
oddsSelection = event.xpath('selections//selection')
home_price = draw_price = away_price = 0.00
for selection in oddsSelection:
selType = take_first(selection.xpath('hadvalue/text()').extract())
odd_up = take_first(selection.xpath('currentpriceup/text()').extract())
odd_down = take_first(selection.xpath('currentpricedown/text()').extract())
if odd_up and odd_down:
if selType == u'H':
home_price = odd_up+'/'+odd_down
elif selType == u'D':
draw_price = odd_up+'/'+odd_down
elif selType == u'A':
away_price = odd_up+'/'+odd_down
MOdict['runners'] = [{'runnerName': 'HOME',
'price': home_price},
{'runnerName': 'DRAW',
'price': draw_price},
{'runnerName': 'AWAY',
'price': away_price},
]
l.add_value('markets', [MOdict, ])
items.append(l.load_item())
return items
def createMOonly(event, response, bookieName):
'''
Take event selection, parse dateTime, teams
, MO prices then return event loaded.
(Remember to load item one of the parse methods of spider class
must return it, but this would terminate current chain, so we pass
it along in a list, this function just create the item for appending
to that list)
'''
teams = []
eventName = take_first(event.xpath('td[1]/text()').extract())
if eventName:
teams = eventName.lower().split(' - ')
dateTime = take_first(event.xpath('td[2]/text()').extract())
home_price = take_first(event.xpath('td[3]/a/text()').extract())
draw_price = take_first(event.xpath('td[4]/a/text()').extract())
away_price = take_first(event.xpath('td[5]/a/text()').extract())
mDict = {'marketName': 'Match Odds',
'runners': [{'runnerName': 'HOME',
'price': home_price},
{'runnerName': 'DRAW',
'price': draw_price},
{'runnerName': 'AWAY',
'price': away_price}
]
}
# Load the MO data
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', bookieName)
l.add_value('dateTime', dateTime)
l.add_value('teams', teams)
l.add_value('markets', [mDict, ])
# Debug
log.msg('No more link for %s, load MO prices only' % eventName, level=log.DEBUG)
log.msg('Datetime: %s' % dateTime, level=log.DEBUG)
log.msg('Teams %s' % teams, level=log.DEBUG)
log.msg('mDict: %s' % mDict, level=log.DEBUG)
# Return loaded item
return l.load_item()
def parseData(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(
response.xpath('//div[@class="eventSubHeader"]/'
'span/script/text()').extract())
dateTime = dateTime[dateTime.find('\"') + 1:dateTime.find('\")')]
l.add_value('dateTime', dateTime)
eventName = take_first(
response.xpath('//div[@id="contentHeader"]/'
'h3/text()').extract())
if eventName:
teams = eventName.lower().split(' v ')
l.add_value('teams', teams)
# Markets
mkts = response.xpath('//div[starts-with(@id, "mk_")]')
allmktdicts = []
for mkt in mkts:
marketName = take_first(mkt.xpath('ul/li/a/text()').extract())
mdict = {'marketName': marketName, 'runners': []}
runners = mkt.xpath('ul/li/ul/li/div[@class="fOutcome"]')
for runner in runners:
runnername = take_first(
runner.xpath('div[@class="outName"]/text()').extract())
price = take_first(
runner.xpath('div[@class="outBet"]/a/text()').extract())
mdict['runners'].append({
'runnerName': runnername,
'price': price
})
allmktdicts.append(mdict)
# Do some Betinternet specific post processing and formating
for mkt in allmktdicts:
if '90 mins' in mkt['marketName']:
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if teams[0] in runner['runnerName'].lower():
runner['runnerName'] = 'HOME'
elif teams[1] in runner['runnerName'].lower():
runner['runnerName'] = 'AWAY'
elif 'Draw' in runner['runnerName']:
runner['runnerName'] = 'DRAW'
elif 'Correct Score' in mkt['marketName']:
for runner in mkt['runners']:
if teams[1] in runner['runnerName'].lower():
runner['reverse_tag'] = True
else:
runner['reverse_tag'] = False
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parseData(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(
response.xpath('//table[@class="liveEventInfo"]/tr/td/'
'h1/span[@class="clock"]/text()').extract())
# Remove the (#686) stuff
dateTime = dateTime[:dateTime.find('(')]
l.add_value('dateTime', dateTime)
eventName = take_first(
response.xpath('//table[@class="liveEventInfo"]/'
'tr/td/h1/text()').extract())
if eventName:
teams = eventName.lower().split('vs.')
l.add_value('teams', teams)
# Markets
marketNames = response.xpath(
'//div[@id="event-filters"]/'
'h3[@class="capsTableHead"]/text()').extract()
# Sometimes a single row head gives an extra blank
marketNames = [mName.strip() for mName in marketNames if mName.strip()]
marketContent = response.xpath('//div[@id="event-filters"]/table')
mkts = zip(marketNames, marketContent)
allmktdicts = []
for mkt in mkts:
marketName = mkt[0]
mdict = {'marketName': marketName, 'runners': []}
runners = mkt[1].xpath('tr/td')
for runner in runners:
runnername = runner.xpath('./text()').extract()
runnername = take_first(
[rName.strip() for rName in runnername if rName.strip()])
handicapname = take_first(
runner.xpath('em[@class="purple"]/text()').extract())
if handicapname:
runnername += ' ' + handicapname.strip()
price = take_first(runner.xpath('span/text()').extract())
mdict['runners'].append({
'runnerName': runnername,
'price': price
})
allmktdicts.append(mdict)
# Do some Tonybet specific post processing and formating
for mkt in allmktdicts:
if '2-3 Way' in mkt['marketName']:
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if teams[0].strip() in runner['runnerName'].lower():
runner['runnerName'] = 'HOME'
elif teams[1].strip() in runner['runnerName'].lower():
runner['runnerName'] = 'AWAY'
elif 'X' == runner['runnerName']:
runner['runnerName'] = 'DRAW'
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parseData(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
# Once against resp is js array
# jsonstr = re.sub("[,]{2,}", ",", response.body)
jsonstr = re.sub(",(?=,)", ",null", response.body)
try:
jsonBody = json.loads(jsonstr)
except ValueError as e:
log.msg(e, level=log.ERROR)
log.msg('jsonstr: %s' % jsonstr)
return []
items = []
for event in response.meta['events'].values():
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = event['dateTime']
l.add_value('dateTime', dateTime)
teams = event['teams']
if teams:
l.add_value('teams', teams)
# Get odds
MOdict = {'marketName': 'Match Odds', 'runners': []}
for mktData in jsonBody:
marketId = str(mktData[0])
# To upgrade in future could be marketData[0] in
# event['marketIds']
if marketId == event['MOmarketId']:
# Is the assumption 1x2 is first always good?
oddsData = mktData[2][0][1] # 1x2 data is first
if oddsData:
home_price = am2dec(oddsData[1])
draw_price = am2dec(oddsData[3])
away_price = am2dec(oddsData[5])
MOdict['runners'] = [
{
'runnerName': 'HOME',
'price': home_price
},
{
'runnerName': 'DRAW',
'price': draw_price
},
{
'runnerName': 'AWAY',
'price': away_price
},
]
# Add markets
l.add_value('markets', [
MOdict,
])
# Load item
items.append(l.load_item())
return items
def parse_Data(self, response):
'''
All the data is returned for different markets here, not just 1x2,
so in theory should be relatively easy to upgrade to more markets
'''
# Get the league data we passed as arg through meta
league_name = response.meta['league_name']
log.msg('Going to parse data for league: %s' % league_name)
# Parse JSON from body
try:
jResp = json.loads(response.body)
except ValueError as e:
log.msg(e, level=log.ERROR)
log.msg('Response body not JSON for league: %s' % league_name)
log.msg('response.body dump: \n%s' % response.body)
# Traverse JSON
try:
if 'mod' in jResp.keys():
jsonMOD = json.loads(jResp['mod'])
else:
log.msg("'mod' key not found for league: %s" % league_name)
log.msg('jResp key dump: %s' % jResp.keys())
return []
# Data
if 'd' in jsonMOD.keys():
jsonData = jsonMOD['d']
else:
log.msg("'d' key not found for league: %s" % league_name)
log.msg('jsonMOD key dump: %s ' % jsonMOD.keys())
return []
# Competitions, it seems if no 1x2 there won't be a c key.
if 'c' in jsonData.keys():
jsonComps = jsonData['c']
else:
log.msg("'c' key not found for league: %s" % league_name)
log.msg('jsonData key dump: %s' % jsonData.keys())
# It looks like jsonMOD['bt'] lists the mkt types avail for sel btn at top,so check with that:
log.msg("No 1x2 mkt? jsonMOD['bt'] check: %s" % jsonMOD['bt'])
return []
# Events (I load the only one comp at once so safe to take zeroth)
jsonEvents = jsonComps[0]['e']
except KeyError as e:
log.msg(e, level=log.ERROR)
log.msg('Error response or 404? No 1x2 mkts? For league: %s' % league_name)
return []
items = []
for jsonEvent in jsonEvents:
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = jsonEvent['i'][4] # u'09 / 06'
l.add_value('dateTime', dateTime)
homeName = jsonEvent['i'][0]
awayName = jsonEvent['i'][1]
if homeName and awayName:
teams = [homeName, awayName]
l.add_value('teams', teams)
# MO prices
MOdict = {'marketName': 'Match Odds'}
home_price = jsonEvent['o']['1x2'][1]
draw_price = jsonEvent['o']['1x2'][5]
away_price = jsonEvent['o']['1x2'][3]
MOdict['runners'] = [{'runnerName': 'HOME',
'price': home_price},
{'runnerName': 'DRAW',
'price': draw_price},
{'runnerName': 'AWAY',
'price': away_price},
]
# Add markets
l.add_value('markets', [MOdict, ])
# Load item
items.append(l.load_item())
return items
def parseData(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
# First row is just league heading,and you want to exclude
# 'sep' class rows.
eventSelection = response.xpath(
'//table[@class="ComingUPTable"]/tr'
'[position()>1][not(@class="moreEvent")]')
items = []
for event in eventSelection:
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(event.xpath('td[2]/text()').extract())
l.add_value('dateTime', dateTime)
eventName = take_first(event.xpath('td[3]/div/a/text()').extract())
if eventName:
teams = eventName.lower().split(' - ')
l.add_value('teams', teams)
# MO prices
MOdict = {'marketName': 'Match Odds'}
home_price = take_first(
event.xpath('td[starts-with(@class, "home")]/'
'div[@class="val"]/a/text()').extract())
draw_price = take_first(
event.xpath('td[starts-with(@class, "draw")]/'
'div[@class="val"]/a/text()').extract())
away_price = take_first(
event.xpath('td[starts-with(@class, "away")]/'
'div[@class="val"]/a/text()').extract())
MOdict['runners'] = [
{
'runnerName': 'HOME',
'price': home_price
},
{
'runnerName': 'DRAW',
'price': draw_price
},
{
'runnerName': 'AWAY',
'price': away_price
},
]
# Add markets
l.add_value('markets', [
MOdict,
])
# Load item
items.append(l.load_item())
return items
def parse_Data(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
day = take_first(
response.xpath(
'//div[@class="section first"]/div[@class="date"]/'
'div[@class="cal"]/span[@class="day"]/text()').extract())
month = take_first(
response.xpath('//div[@class="section first"]/'
'div[@class="date"]/div[@class="cal"]/'
'span[@class="month"]/text()').extract())
if day and month and 'back' not in month:
# Weirdly sometimes month appears as 'back'
dateTime = day + ' ' + month
else:
# Other format, e.g. Saturday
dateTime = take_first(
response.xpath('//div[@class="section first"]/'
'div[@class="date"]/text()').extract())
l.add_value('dateTime', dateTime)
teams = response.xpath(
'//div[@id="eventDetailTitle"]/h3/span[@class="name"]/text()'
).extract()
l.add_value('teams', teams)
# Markets
mkts = response.xpath('//table[starts-with(@class, "oddTbl")]')
allmktdicts = []
for mkt in mkts:
marketName = take_first(
mkt.xpath('tr[@class="name"]/th/text()').extract())
mdict = {'marketName': marketName, 'runners': []}
if 'Correct Score' in marketName or 'Halftime / Fulltime' in marketName:
runners = mkt.xpath(
'tr[not(starts-with(@class, "result")) and not(@class="name")]/td/a'
)
for runner in runners:
runnerName = take_first(
runner.xpath('span[@class="name"]/text()').extract())
price = take_first(
runner.xpath('span[@class="odd"]/text()').extract())
mdict['runners'].append({
'runnerName': runnerName,
'price': price
})
else:
runnerNames = mkt.xpath(
'tr[starts-with(@class, "result")]/th/text()').extract()
if not runnerNames:
# Sometimes a span
runnerNames = mkt.xpath(
'tr[starts-with(@class, "result")]/th/span/text()'
).extract()
prices = mkt.xpath(
'tr[not(starts-with(@class, "result")) and not(@class="name")]/'
'td/a/text()').extract()
runners = zip(runnerNames, prices)
for runner in runners:
mdict['runners'].append({
'runnerName': runner[0],
'price': runner[1]
})
allmktdicts.append(mdict)
# Do some Doxxbet specific post processing and formating
for mkt in allmktdicts:
if 'Result' == mkt['marketName']:
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if runner['runnerName'] == u'1':
runner['runnerName'] = 'HOME'
elif runner['runnerName'] == u'2':
runner['runnerName'] = 'AWAY'
elif runner['runnerName'] == u'X':
runner['runnerName'] = 'DRAW'
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parseEvent(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(
response.xpath(
'//div[@class="event-markets"]/div[@class="block-header"]'
'/div[@class="center-col-options"]'
'/div[last()]/text()').extract())
l.add_value('dateTime', dateTime)
eventName = take_first(
response.xpath('//div[@class="game-all-markets"]/div/'
'div[@class="block-header"]/'
'span[@class="block-title"]/text()').extract())
if eventName:
teams = eventName.lower().split(' v ')
l.add_value('teams', teams)
# I think we need to more specifically target markets formatting too diff
# between them
markets = response.xpath('//div[@class="market-wrapper"]')
# Lets get MO
# N.B. just like a filesystem path we go upward with '..', after
# we find the ones with Match Result only
MOmarket = markets.xpath(
'div[@class="block-header-wrapper"]/div/'
'span[@class="block-title" and text()="Match Result"]/'
'../../../div[@class="matches"]')
# MO prices
MOdict = {'marketName': 'Match Odds'}
home_price = take_first(
MOmarket.xpath('table/tr[@class="body-row"]/td[1]//'
'span[@class="odds-fractional"]/text()').extract())
draw_price = take_first(
MOmarket.xpath('table/tr[@class="body-row"]/td[2]//'
'span[@class="odds-fractional"]/text()').extract())
away_price = take_first(
MOmarket.xpath('table/tr[@class="body-row"]/td[3]//'
'span[@class="odds-fractional"]/text()').extract())
MOdict['runners'] = [
{
'runnerName': 'HOME',
'price': home_price
},
{
'runnerName': 'DRAW',
'price': draw_price
},
{
'runnerName': 'AWAY',
'price': away_price
},
]
# CS market
CSmarket = markets.xpath(
'div[@class="block-header-wrapper"]/div/'
'span[@class="block-title" and text()="Correct Score"]/'
'../../../div[@class="football-matches"]')
CSresults = CSmarket.xpath(
'div[starts-with(@class, "match-row-grouped ")]/'
'div[@class="match featured-match featured-match-row"]/'
'div')
CSdict = {'marketName': 'Correct Score', 'runners': []}
for result in CSresults:
runnerName = take_first(
result.xpath('div[@class="bet-title-3row"]/text()').extract())
price = take_first(
result.xpath(
'div[@class="home-odds"]/div/'
'span[@class="odds-fractional"]/text()').extract())
if runnerName and price:
if teams[1] in runnerName.lower():
# Tag for score reversing by loader (e.g. if Team2 1-0 want
# just 0-1 to match Betfair format and avoid team name comp)
CSdict['runners'].append({
'runnerName': runnerName,
'price': price,
'reverse_tag': True
})
else:
CSdict['runners'].append({
'runnerName': runnerName,
'price': price,
'reverse_tag': False
})
# Add markets
l.add_value('markets', [MOdict, CSdict])
# Load item
return l.load_item()
def parse_Data(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(
response.xpath('//div[@class="stats-time"]/'
'span[@class="time"]/text()').extract())
l.add_value('dateTime', dateTime)
eventName = take_first(
response.xpath('//div[starts-with(@class, "title ev ev-")]/'
'span/text()').extract())
if eventName:
teams = eventName.lower().split(' v ')
l.add_value('teams', teams)
# Markets (only from main area)
mkts = response.xpath(
'//div[@id="main-area"]//div[starts-with(@class, "expander") and @data-mkt_id]'
)
allmktdicts = []
for mkt in mkts:
marketName = take_first(
mkt.xpath('h6/span[@class="mkt-name"]/text()').extract())
if not marketName:
# Some are not in span
marketName = take_first(
mkt.xpath('h6/text()').extract()).strip()
mdict = {'marketName': marketName, 'runners': []}
# N.B. The collapsed markets need seperate AJAX requests to grab data
# so far now just forget about them (Also sometimes the mkts, e.g. asians
# are ul/li instead of tables).
runners = mkt.xpath(
'div[@class="expander-content"]/table/'
'tbody/tr[@class="limited-row"]/td[starts-with(@class, "seln")]'
)
if not runners:
# Sometimes tr has no class
runners = mkt.xpath('div[@class="expander-content"]/table/'
'tbody/tr/td[starts-with(@class, "seln")]')
for runner in runners:
runnername = take_first(
runner.xpath(
'.//span[@class="seln-name"]/span/text()').extract())
if not runnername:
# E.g. for Correct Score
runnername = take_first(
runner.xpath('.//span[@class="seln-sort"]/span/text()'
).extract())
price = take_first(
runner.xpath(
'.//span[@class="price dec"]/text()').extract())
mdict['runners'].append({
'runnerName': runnername,
'price': price
})
allmktdicts.append(mdict)
# Do some Titanbet specific post processing and formating
for mkt in allmktdicts:
if 'Full Time Result' == mkt['marketName']:
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if teams[0] in runner['runnerName'].lower():
runner['runnerName'] = 'HOME'
elif teams[1] in runner['runnerName'].lower():
runner['runnerName'] = 'AWAY'
elif 'X' == runner['runnerName']:
runner['runnerName'] = 'DRAW'
elif 'Correct Score' in mkt['marketName']:
for runner in mkt['runners']:
if teams[1] in runner['runnerName'].lower():
runner['reverse_tag'] = True
else:
runner['reverse_tag'] = False
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def pre_parse_Data(self, response):
base_url = 'http://www.fortunawin.com'
headers = {
'Host': 'www.fortunawin.com',
'Referer': response.url,
'X-Requested-With': 'XMLHttpRequest'
}
events = response.xpath('//tr[starts-with(@id, "bet-")]')
for event in events:
moreLink = take_first(
event.xpath(
'td[@class="col_toggle"]/'
'a[starts-with(@id, "toggler-id-")]/@href').extract())
dateTime = take_first(
event.xpath(
'.//span[starts-with(@id, "datetime-")]/text()').extract())
dateTime = take_first(dateTime.strip().replace('.',
'/').rsplit('/', 1))
eventName = take_first(
event.xpath('.//td[@class="col_title"]/'
'div/span/a/text()').extract())
# print eventName
# if 'Antalyaspor' in eventName:
# print response.url
# stop = raw_input('e2c')
MOdict = {'marketName': 'Match Odds', 'runners': []}
home_price = take_first(
event.xpath('td[2]/a/@data-rate').extract())
draw_price = take_first(
event.xpath('td[3]/a/@data-rate').extract())
away_price = take_first(
event.xpath('td[4]/a/@data-rate').extract())
MOdict['runners'] = [
{
'runnerName': 'HOME',
'price': home_price
},
{
'runnerName': 'DRAW',
'price': draw_price
},
{
'runnerName': 'AWAY',
'price': away_price
},
]
eventDic = {
'eventName': eventName,
'dateTime': dateTime,
'MOdict': MOdict
}
if moreLink:
yield Request(url=base_url + moreLink + '&ajax=1',
headers=headers,
meta=eventDic,
callback=self.parse_Data)
else:
# Write the MO market alone
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
if eventName:
teams = eventName.lower().split('-')
l.add_value('teams', teams)
l.add_value('dateTime', dateTime)
l.add_value('markets', [
MOdict,
])
# Load item
yield l.load_item()
def parse_Data(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
# Betway has very annoying display for each league,
# with all events of certain date under that date block head.
tableRows = response.xpath('//tbody[@class="oddsbody"]/tr')
# For all rows under with no date will use latest blockdate,
# single item list to match xpath extract
blockdate = []
items = []
for row in tableRows:
# Test with xpath if date row by seeing if td has parent
# tr with class date. Surely easier way to just check self?
if take_first(
row.xpath('@class').extract()) and 'date' in take_first(
row.xpath('@class').extract()):
blockdate = row.xpath('td/text()').extract()
else:
# Else test if this is an 'event' row, using time as
# criteria.
rowtime = row.xpath('td[@class="market_title"]/div[1]/text()'
).extract() # 24hr
if rowtime != []:
# We have event.
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(blockdate)
l.add_value('dateTime', dateTime)
eventName = take_first(
row.xpath('td[@class="market_title"]/'
'a/text()').extract())
if eventName:
teams = eventName.lower().split(' - ')
l.add_value('teams', teams)
# MO prices
MOdict = {'marketName': 'Match Odds'}
home_price = take_first(
row.xpath(
'td[contains(@class,"outcome")][1]/'
'a[@class="outcome"]/'
'div[@class="outcome_button"]/text()').extract())
draw_price = take_first(
row.xpath(
'td[contains(@class,"outcome")][2]/'
'a[@class="outcome"]/'
'div[@class="outcome_button"]/text()').extract())
away_price = take_first(
row.xpath(
'td[contains(@class,"outcome")][3]/'
'a[@class="outcome"]/'
'div[@class="outcome_button"]/text()').extract())
MOdict['runners'] = [
{
'runnerName': 'HOME',
'price': home_price
},
{
'runnerName': 'DRAW',
'price': draw_price
},
{
'runnerName': 'AWAY',
'price': away_price
},
]
# Add markets
l.add_value('markets', [
MOdict,
])
# Load item
items.append(l.load_item())
return items
def parse_match(self, response):
eventSelection = response.xpath('//table[@class="tableData"]/tbody/tr')
if (response.url.endswith('Outright.html')
or response.url.endswith('Trophy.html')):
# Not interested, bail.
# Can't filter this earlier, because of redirect name change of link.
return []
items = []
for event in eventSelection:
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
if response.meta.get('redirect_times') is not None:
# WilliamHill redirects for markets with single event, must handle differently.
log.msg('Scraping redirected site: %s' % response.url)
dateTime = take_first(
response.xpath('//div[@id="contentHead"]/'
'span[@id="eventDetailsHeader"]/nobr/'
'span/text()').extract())
if not dateTime:
dateTime = ''
l.add_value('dateTime', dateTime)
team1 = take_first(
event.xpath('td[1]/div/div[@class="eventselection"]/text()'
).extract())
team2 = take_first(
event.xpath('td[3]/div/div[@class="eventselection"]/text()'
).extract())
if team1 and team2:
l.add_value('teams', [team1, team2])
else:
dateTime = take_first(
event.xpath('td[1]/span/text()|td[1]/text()').extract())
if not dateTime:
# Could be Live
dateTime = ''
l.add_value('dateTime', dateTime)
eventName = take_first(
event.xpath('td[3]/a/span/text()').extract())
if eventName:
teams = eventName.lower().split(' v ')
l.add_value('teams', teams)
# MO market is same either way
MOdict = {'marketName': 'Match Odds'}
home_price = take_first(
event.xpath(
'td[5]/div/div[@class="eventprice"]/text()').extract())
draw_price = take_first(
event.xpath(
'td[6]/div/div[@class="eventprice"]/text()').extract())
away_price = take_first(
event.xpath(
'td[7]/div/div[@class="eventprice"]/text()').extract())
MOdict['runners'] = [
{
'runnerName': 'HOME',
'price': home_price
},
{
'runnerName': 'DRAW',
'price': draw_price
},
{
'runnerName': 'AWAY',
'price': away_price
},
]
# Add markets
l.add_value('markets', [
MOdict,
])
# Load item
items.append(l.load_item())
return items
def parseEvent(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(
response.xpath('//table[starts-with(@id, "TBL_Content")]/'
'tr[1]/td[@class="playtime"]/p/text()').extract())
l.add_value('dateTime', dateTime)
eventName = take_first(
response.xpath('//td[@class="colmiddle"]/div[@class="bets shadow"]'
'/h2[@class="header gradient"]/'
'span[@class="text"]/span/text()').extract())
if eventName:
teams = eventName.lower().split(' - ')
l.add_value('teams', teams)
# MO market first
# Get table associated with the td that has a p with text "Match"
# (remember xpath .. means move up just like linux fs)
MOdict = {'marketName': 'Match Odds'}
MOtable = response.xpath('//tr/td[@class="bets"]/p[@class="info"]'
'[text()="Match"]/../table')
home_price = take_first(
MOtable.xpath('tr/td[1]/p/strong/text()').extract())
draw_price = take_first(
MOtable.xpath('tr/td[2]/p/strong/text()').extract())
away_price = take_first(
MOtable.xpath('tr/td[3]/p/strong/text()').extract())
MOdict['runners'] = [
{
'runnerName': 'HOME',
'price': home_price
},
{
'runnerName': 'DRAW',
'price': draw_price
},
{
'runnerName': 'AWAY',
'price': away_price
},
]
# CS market
CSdict = {'marketName': 'Correct Score', 'runners': []}
CStable = response.xpath('//tr/td[@class="bets"]/p[@class="info"]'
'[text()="Correct Score"]/../table')
CSresults = CStable.xpath('tr/td/p')
for result in CSresults:
# Deal with ':' in name in loaders
runnerName = take_first(result.xpath('span/text()').extract())
price = take_first(result.xpath('strong/text()').extract())
CSdict['runners'].append({
'runnerName': runnerName,
'price': price,
'reverse_tag': False
})
# Add markets
l.add_value('markets', [MOdict, CSdict])
# Load item
return l.load_item()
def parse_events(self, response):
'''
Loop over all events. If no moreLink
just yield the match odds data, if moreLink
pass the MO data in meta and then request other markets
data, finally adding all mkts in parseData
'''
pageNum = response.meta['page']
log.msg('Parsing events for page: %s' % pageNum)
try:
jResp = json.loads(response.body)
sel = Selector(text=jResp['selections'])
except ValueError as e:
log.msg('Error could not load JSON when paging? %s' % e,
level=log.ERROR)
base_url = 'https://www.nordicbet.com/eng/sportsbook'
headers = {
'Host': 'www.nordicbet.com',
'Referer': 'https://www.nordicbet.com/eng/sportsbook',
'X-Requested-With': 'XMLHttpRequest'
}
events = sel.xpath('//table[starts-with(@class, "resultstbl 1x2")]')
for event in events:
# Does event have moreLink?
moreLink = take_first(
event.xpath('tr/td/div[@class="extra"]/a').extract())
# eventId needed for call for other markets (only if moreLink)
eventClass = take_first(event.xpath('@class').extract())
eId = eventClass.rsplit('_')[-1]
# Build MO market and other event details
# Get MO prices only and add to items ready to passed in meta
# for insertion in parseData (first iteration of loop only)
teams = []
eventName = event.xpath(
'.//tr/td/div[@class="label"]/text()'
'| .//tr/td/div[@class="label"]/a/text()').extract()
eventName = ''.join([eN.strip() for eN in eventName])
if eventName:
teams = eventName.lower().split(' - ')
dateTime = take_first(
event.xpath('.//tr/td//span[@class="day"]/text()').extract())
if not dateTime:
# Could be live
livelogo = event.xpath(
'.//tr/td[2]/div[@class="endingtime"]/div[@class="livebet_now_logo"]'
)
if livelogo:
dateTime = 'Today'
else:
log.msg('Error no dateTime could be grabbed')
home_price = take_first(
event.xpath(
'.//tr/td[4]//div[@class="odds_middle"]/text()').extract())
draw_price = take_first(
event.xpath(
'.//tr/td[5]//div[@class="odds_middle"]/text()').extract())
away_price = take_first(
event.xpath(
'.//tr/td[6]//div[@class="odds_middle"]/text()').extract())
mDict = {
'marketName':
'Match Odds',
'runners': [{
'runnerName': 'HOME',
'price': home_price
}, {
'runnerName': 'DRAW',
'price': draw_price
}, {
'runnerName': 'AWAY',
'price': away_price
}]
}
if moreLink:
# Pass along MOmarket and other event details, and make
# request for more markets
stamp = int(time.time() * 1000)
meta = {
'teams': teams,
'eventName': eventName,
'dateTime': dateTime,
'mDict': mDict
}
# Make request for event detail
GETstr = ('?cmd=chooseInAjax&sidebet_type=1x2&show_extra=true&'
'game_id=%s&source=15&_=%s' % (eId, stamp))
yield Request(url=base_url + GETstr,
headers=headers,
meta=meta,
dont_filter=True,
callback=self.parseData)
else:
# Just load the MO market only event
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
l.add_value('dateTime', dateTime)
l.add_value('teams', teams)
l.add_value('markets', [
mDict,
])
# Debug
log.msg('No moreLink for %s, load MO prices only' % eventName,
level=log.DEBUG)
# log.msg('Datetime: %s' % dateTime, level=log.DEBUG)
# log.msg('Teams %s' % teams, level=log.DEBUG)
# log.msg('mDict: %s' % mDict, level=log.DEBUG)
# Return loaded item
# stop = raw_input('e2c')
yield l.load_item()
def parseData(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = response.meta['dateTime']
l.add_value('dateTime', dateTime)
teams = response.meta['teams']
l.add_value('teams', teams)
# Markets
# There is not enough regularity for one-size-fits-all but could we
# at least grab markets in groups sharing similar formatting (CS and
# HT/FT for example)
mktBlocks = response.xpath(
'//table[starts-with(@class, "resultstbl")]')
allmktdicts = [
response.meta['mDict'],
] # Add in MO from meta
underover_re = re.compile("Under/over: (?P<val>\d+\.\d+)")
for mktBlock in mktBlocks:
blockName = take_first(
mktBlock.xpath('tr[1]/td[1]/div/text()').extract())
if 'Under/over' in blockName:
# Things are tricky because 'Under/over' or 'Under/over team'
# is the grouping, but the marketName should actually be the row
# label, i.e. 'Under/over 0.5', with runners 'Under 0.5', 'Over
# 0.5'
rows = mktBlock.xpath('tr[re:test(@class, "item$")]')
for row in rows:
marketName = take_first(
row.xpath('td[@class="label"]/div/text()').extract())
# If Under/over teams
if 'team' in blockName:
team = take_first(
row.xpath('td[@align="left"]/text()').extract())
marketName = team + marketName
mDict = {'marketName': marketName, 'runners': []}
# Get the numeric over/under value
r = underover_re.search(marketName)
underover_val = r.groupdict()['val']
# Prices
uprice = take_first(
row.xpath(
'td[@class="bet"]//div[@class="odds_middle"]/text()'
).extract())
oprice = take_first(
row.xpath(
'td[@class="bet last"]//div[@class="odds_middle"]/text()'
).extract())
mDict['runners'].extend([{
'runnerName': 'Under %s' % underover_val,
'price': str(uprice),
}, {
'runnerName': 'Over %s' % underover_val,
'price': str(oprice)
}])
allmktdicts.append(mDict)
if 'Exact score' == blockName or 'Halftime/fulltime' == blockName:
# N.B. Even half-time exact score different format!
if 'Exact score' == blockName:
marketName = 'Correct Score'
elif 'Halftime/fulltime' == blockName:
marketName = 'Halftime/Fulltime'
mDict = {'marketName': marketName, 'runners': []}
runners = mktBlock.xpath('tr[re:test(@class, "item$")]/'
'td[@class="exact_result"]/'
'table/tr/td[@align="center"]/'
'table')
for runner in runners:
runnerName = take_first(
runner.xpath('td[1]/text()').extract())
price = take_first(
runner.xpath('td[2]//div[@class="odds_middle"]/'
'text()').extract())
mDict['runners'].append({
'runnerName': runnerName,
'price': price
})
allmktdicts.append(mDict)
# # Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parse_Data(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
eventSelection = response.xpath(
'//table[contains(@class, "event-type")]/tbody/tr')
items = []
for event in eventSelection:
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(event.xpath('td[2]/text()').extract())
# Sometimes just a time 13:30 if date is Today, the following -> ''
if dateTime:
dateTime = dateTime.strip()[:-5]
else:
dateTime = 'Today'
if not dateTime or dateTime == ' ':
dateTime = 'Today'
l.add_value('dateTime', dateTime)
eventName = take_first(
event.xpath('td[@class="event"]/a/span/text()').extract())
if eventName:
teams = eventName.lower().split(' v ')
l.add_value('teams', teams)
# MO prices
MOdict = {'marketName': 'Match Odds'}
home_price = take_first(
event.xpath('td[5][@class="price"]/a/strong/text()').extract())
draw_price = take_first(
event.xpath('td[6][@class="price"]/a/strong/text()').extract())
away_price = take_first(
event.xpath('td[7][@class="price"]/a/strong/text()').extract())
MOdict['runners'] = [
{
'runnerName': 'HOME',
'price': home_price
},
{
'runnerName': 'DRAW',
'price': draw_price
},
{
'runnerName': 'AWAY',
'price': away_price
},
]
# Add markets
l.add_value('markets', [
MOdict,
])
# Load item
items.append(l.load_item())
return items
def parse_Data(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
# Unlike most scrapes there is no detail page for each event, just a
# listing of markets. We grab the MO, totals and goals type markets for
# each event (each triplet), but this event may be repeated again with
# further, different totals or goals markets, so at the end we have to
# consolidate markets to single event. Also half time versions are dealt
# with.
rows = response.xpath(
'//tr[@class="TrGameOdd" or @class="TrGameEven"]')
events = list(chunks(rows, 3))
eventDics = []
for evt in events:
eventDic = {}
team1 = take_first(evt[0].xpath('td[4]/text()').extract())
team2 = take_first(evt[1].xpath('td[4]/text()').extract())
teams = [team1, team2]
eventDic['teams'] = teams
dateTime = take_first(evt[0].xpath('td[2]/text()').extract())
eventDic['dateTime'] = dateTime
eventDic['markets'] = []
# This regex matches names like o3.75, u.75, o2 and prices like +110
# or EV.
Tregex = re.compile(
"(?P<name>[ou](\d+)?(.\d+)?)(?P<price>([+-]\d+|EV))")
# Similar to Tregex but also matches PK as name.
Gregex = re.compile(
"(?P<name>([+-](\d+)?(.\d+)?|PK))(?P<price>([+-]\d+|EV))")
for ind in ['5', '6', '7']:
# index 5 in MO, 6 is totals, 7 is goals
# NB for Totals and Goals (i.e. ind 6 or 7) the home_price and
# away_price are really combinations of name and price.
home_price = take_first(evt[0].xpath('td[' + ind +
']/text()').extract())
away_price = take_first(evt[1].xpath('td[' + ind +
']/text()').extract())
draw_price = take_first(evt[2].xpath('td[' + ind +
']/text()').extract())
if ind == '5' and home_price and away_price and draw_price:
marketName = 'Match Odds'
elif ind == '6' and home_price and away_price:
marketName = 'Totals'
elif ind == '7' and home_price and away_price:
marketName = 'Goals'
else:
continue
# Make sure half time markets have this in marketName, then
# remove it from team names themselves
if '1ST HALF' in team1 or ' 1ST HALF' in team2:
marketName += ' 1ST HALF'
eventDic['teams'] = [
n.replace(' 1ST HALF', '') for n in teams
]
if 'Match Odds' in marketName:
mdict = {'marketName': marketName, 'runners': []}
mdict['runners'] = [
{
'runnerName': 'HOME',
'price': am2dec(home_price)
},
{
'runnerName': 'DRAW',
'price': am2dec(draw_price)
},
{
'runnerName': 'AWAY',
'price': am2dec(away_price)
},
]
eventDic['markets'].append(mdict)
else:
home_price = home_price.encode('utf8').replace('\xc2\xbd', '.5') \
.replace('\xc2\xbc', '.25') \
.replace('\xc2\xbe', '.75')
away_price = away_price.encode('utf8').replace('\xc2\xbd', '.5') \
.replace('\xc2\xbc', '.25') \
.replace('\xc2\xbe', '.75')
regex = None
if ind == '6':
# Totals
regex = Tregex
elif ind == '7':
# Goals
regex == Gregex
if regex:
home = regex.search(home_price)
away = regex.search(away_price)
if home and away:
home_name = home.groupdict()['name']
home_price = am2dec(home.groupdict()['price'])
away_name = away.groupdict()['name']
away_price = am2dec(away.groupdict()['price'])
marketName += ' ' + home_name + '/' + away_name
mdict = {'marketName': marketName, 'runners': []}
mdict['runners'] = [
{
'runnerName': home_name,
'price': home_price
},
{
'runnerName': away_name,
'price': away_price
},
]
eventDic['markets'].append(mdict)
else:
log.msg('No regex match: home: %s, away:%s' %
(home_name, away_name),
level=log.ERROR)
log.msg('Teams: %s, %s' % (team1, team2),
level=log.ERROR)
eventDics.append(eventDic)
# Finally consolidate all markets with same teams and date.
newDics = []
for eD1 in eventDics:
new_eD = {
'teams': eD1['teams'],
'dateTime': eD1['dateTime'],
'markets': []
}
mkts = [
e['markets'] for e in eventDics
if e['teams'] == eD1['teams']
and e['dateTime'] == eD1['dateTime']
]
# flatten
mkts = [item for sublist in mkts for item in sublist]
new_eD['markets'] = mkts
# if not appended already add it
if ((new_eD['teams'], new_eD['dateTime'])
not in [(d['teams'], d['dateTime']) for d in newDics]):
newDics.append(new_eD)
nevents = []
for nevent in newDics:
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
l.add_value('dateTime', nevent['dateTime'])
l.add_value('teams', nevent['teams'])
l.add_value('markets', nevent['markets'])
nevents.append(l.load_item())
return nevents
def parse_Data(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.context['spiderlocale'] = 'es_ES.utf8' # Set Spanish locale for dates
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = response.meta['dateTime']
l.add_value('dateTime', dateTime)
eventName = take_first(response.xpath('//div[@class="breadcrumbs-content"]/'
'ul/li[@class="last-item"]/a/text()').extract())
if eventName:
teams = eventName.lower().split(' v ')
l.add_value('teams', teams)
# Markets
# N.B. even appending ?show_all=Y to url still means unexpanded markets
# need additional get market requests. Not worth it for now.
mkts = response.xpath('//div[starts-with(@class, "expander ")]')
allmktdicts = []
for mkt in mkts:
marketName = take_first(mkt.xpath('h6/span/text()').extract())
mdict = {'marketName': marketName, 'runners': []}
runners = mkt.xpath('div[@class="expander-content"]/table/tbody/tr/td')
for runner in runners:
runnername = take_first(runner.xpath('div/button/span/'
'span[@class="seln-name" or @class="seln-sort"]/'
'span/text()').extract())
price = take_first(runner.xpath('div/button/span/'
'span[@class="price dec"]/'
'text()').extract())
if runnername and price:
mdict['runners'].append({'runnerName': runnername, 'price': price})
if mdict['marketName'] and mdict['runners']:
allmktdicts.append(mdict)
# Do some Sportium specific post processing and formating
for mkt in allmktdicts:
if '1-X-2' == mkt['marketName']:
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if teams[0] in runner['runnerName'].lower():
runner['runnerName'] = 'HOME'
elif teams[1] in runner['runnerName'].lower():
runner['runnerName'] = 'AWAY'
elif 'X' == runner['runnerName']:
runner['runnerName'] = 'DRAW'
elif 'Marcador exacto' == mkt['marketName']:
mkt['marketName'] = 'Correct Score'
elif 'Descanso/Final del Partido' == mkt['marketName']:
mkt['marketName'] = 'HALF-TIME/FULL-TIME'
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()
def parseEvent(self, response):
log.msg('Going to parse data for treeId: %s' % response.meta['treeId'],
level=log.DEBUG)
# Resp is HTML wrapped in JSON
jResp = json.loads(response.body)
htmlResp = jResp['HTML_ROW_WITH_ADDITIONAL_MARKETS']
# Get selector from this unicode string
sel = Selector(text=htmlResp)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(
sel.xpath('//td[starts-with(@class,"first")]/'
'table/tr//td[@class="date"]/text()').extract())
l.add_value('dateTime', dateTime)
team1 = take_first(
sel.xpath('//td[@class="name" or @class="today-name"]'
'/span/div[1]/text()').extract())
team2 = take_first(
sel.xpath('//td[@class="name" or @class="today-name"]'
'/span/div[2]/text()').extract())
if team1 and team2:
l.add_value('teams', [team1, team2])
# Markets
markets = sel.xpath('//div[@class="block-market-table-wrapper"]'
'/div[@class="market-inline-block-table-wrapper"]')
# MO market first
# Get table associated with the td that has a p with text "Match"
# (remember xpath .. means move up just like linux fs)
MOdict = {'marketName': 'Match Odds'}
# N.B. you may consider using [contains(text(), "Result")], but this
# matches other markets.
MOtable = markets.xpath(
'div[@class="market-table-name"]/'
'div[@class="name-field"][text()=" Result "]/../../table')
home_price = take_first(
MOtable.xpath('tr[1]/td[2]/span/text()').extract())
draw_price = take_first(
MOtable.xpath('tr[2]/td[2]/span/text()').extract())
away_price = take_first(
MOtable.xpath('tr[3]/td[2]/span/text()').extract())
MOdict['runners'] = [
{
'runnerName': 'HOME',
'price': home_price
},
{
'runnerName': 'DRAW',
'price': draw_price
},
{
'runnerName': 'AWAY',
'price': away_price
},
]
# CS market
CSdict = {'marketName': 'Correct Score', 'runners': []}
CStable = markets.xpath(
'div[@class="market-table-name"]/div[@class="name-field"]'
'[text()=" Correct Score "]/../../table')
CShomeRes = CStable.xpath(
'tr/td[1]/div[contains(@class, "coeff-link-3way")]')
CSdrawRes = CStable.xpath(
'tr/td[2]/div[contains(@class, "coeff-link-3way")]')
CSawayRes = CStable.xpath(
'tr/td[3]/div[contains(@class, "coeff-link-3way")]')
for result in CShomeRes:
# Column where home team has winning score (strip brackets in loader)
runnerName = take_first(
result.xpath('div[@class="coeff-handicap"]/text()').extract())
price = take_first(
result.xpath(
'div[@class="coeff-price"]/span/text()').extract())
CSdict['runners'].append({
'runnerName': runnerName,
'price': price,
'reverse_tag': False
})
for result in CSdrawRes:
# Column for draw
runnerName = take_first(
result.xpath('div[@class="coeff-handicap"]/text()').extract())
price = take_first(
result.xpath(
'div[@class="coeff-price"]/span/text()').extract())
CSdict['runners'].append({
'runnerName': runnerName,
'price': price,
'reverse_tag': False
})
for result in CSawayRes:
# Column for away (needs reversing)
runnerName = take_first(
result.xpath('div[@class="coeff-handicap"]/text()').extract())
price = take_first(
result.xpath(
'div[@class="coeff-price"]/span/text()').extract())
CSdict['runners'].append({
'runnerName': runnerName,
'price': price,
'reverse_tag': True
})
# Add markets
l.add_value('markets', [MOdict, CSdict])
# Load item
return l.load_item()
def parseData(self, response):
log.msg('Going to parse data for URL: %s' % response.url[20:],
level=log.INFO)
l = EventLoader(item=EventItem2(), response=response)
l.add_value('sport', u'Football')
l.add_value('bookie', self.name)
dateTime = take_first(response.xpath('//div[@id="ContentWrapper"]/'
'div[@class="antepostContainer"]/'
'div[@class="antepostMarketContainer"]/'
'div[1][@class="antepostEventContainer"]/'
'h4/span/text()').extract())
# Sometimes instead of divs we get an entire reponse with same class/ids
# but using tables/tr/td instead of the divs. We could use element wildcards
# but I think it might be a bit messy.
if not dateTime:
dateTime = take_first(response.xpath('//table[@class="antepostEventContainer"][1]/'
'tr[1]/td[1]/h4/span/text()').extract())
dateTime = dateTime.split('-')[1]
dateTime = dateTime.replace('till', '')
l.add_value('dateTime', dateTime)
eventName = take_first(response.xpath('//h3[@class="antepostHeader"]/text()').extract())
if eventName:
teams = eventName.lower().split(' vs ')
l.add_value('teams', teams)
# Markets
mkts = response.xpath('//div[@class="antepostEventContainer"]')
if not mkts:
mkts = response.xpath('//table[@class="antepostEventContainer"]')
allmktdicts = []
for mkt in mkts:
marketName = take_first(mkt.xpath('h4[@class="antepostEventHeader"]/span/text()').extract())
if not marketName:
marketName = take_first(mkt.xpath('tr/td/h4[@class="antepostEventHeader"]/span/text()').extract())
marketName = take_first(marketName.split(' - '))
mdict = {'marketName': marketName, 'runners': []}
if 'Correct Score' in marketName:
runners = mkt.xpath('div[@class="correctScoreContainer"]/'
'div[not(@class="clear")]/div[starts-with(@class, "selection")]')
if not runners:
runners = mkt.xpath('tr/td/table[@class="correctScoreContainer"]/'
'tr/td/table[not(@class="clear")]/'
'tr/td/table[starts-with(@class, "selection")]/tr/td')
for runner in runners:
runnername = take_first(runner.xpath('span[@class="name"]/text()').extract())
price = take_first(runner.xpath('span[@class="price"]/text()').extract())
mdict['runners'].append({'runnerName': runnername,
'price': price,
})
else:
runners = mkt.xpath('div[@class="antepostSelectionContainer"]/'
'div[starts-with(@class, "antepostSelection")]')
if not runners:
runners = mkt.xpath('tr/td/table[@class="antepostSelectionContainer"]/'
'tr/td/table[starts-with(@class, "antepostSelection")]/tr/td')
for runner in runners:
runnerName = take_first(runner.xpath('span[@class="antepostSelectionName"]/text()').extract())
price = take_first(runner.xpath('span[@class="antepostSelectionOdds"]/text()').extract())
mdict['runners'].append({'runnerName': runnerName, 'price': price})
allmktdicts.append(mdict)
# Do some BGbet specific post processing and formating
for mkt in allmktdicts:
if 'Ninety Minutes' == mkt['marketName']:
mkt['marketName'] = 'Match Odds'
for runner in mkt['runners']:
if teams[0] in runner['runnerName'].lower():
runner['runnerName'] = 'HOME'
elif teams[1] in runner['runnerName'].lower():
runner['runnerName'] = 'AWAY'
elif 'Draw' in runner['runnerName']:
runner['runnerName'] = 'DRAW'
elif 'Correct Scores' in mkt['marketName']:
mkt['marketName'] = 'Correct Score'
for runner in mkt['runners']:
if teams[1] in runner['runnerName'].lower():
runner['reverse_tag'] = True
else:
runner['reverse_tag'] = False
# Add markets
l.add_value('markets', allmktdicts)
# Load item
return l.load_item()