def createWebPage(log:Logger, algoId:int, date:str, league:str=None, \
show:bool=False):
'''
Create a web page for the algo, league and date provided
:param log: a logging object
:param date: include all fixtures from this date and onward
:param league: the subject league, None signifies all available leagues
:param show: displays any tables as HTML when True
'''
log.info('Creating web page for algo <{}>, date <{}> and league <{}>\
'.format(algoId, date, league if league else 'ALL'))
mask = log.getMask()
log.setMask(mask & ~Logger.INFO)
tables = presentFixtures(log, algoId, league)
log.setMask(mask)
groups = ''
for groupId, (name, (fixturesTable, leagueTable, formTable)) \
in enumerate(sorted(tables.items(), key=lambda x : x[0])):
collapsibleTheme = 'c' if fixturesTable.getHighlights() else 'b'
groups += COLLAPSIBLE_GROUP.format(groupName=name, fixturesTable= \
addPriorities(modTable(fixturesTable.asHTML(fullyFormed=False))), \
formTable=modTable(formTable.asHTML(fullyFormed=False)), \
leagueTable=modTable(leagueTable.asHTML(fullyFormed=False)),
groupId=groupId, collapsibleTheme=collapsibleTheme)
html = HTML_HEAD + HTML_BODY.format(groups=groups)
log.debug(html)
with open('web/predictions.html', 'w') as f:
f.write(html)
if show:
webbrowser.open(os.getcwd() + '/web/predictions.html')
def test_info(self):
"""Test cases for info logging
"""
l = Logger()
l.setLogger(self.log)
l.info('test_info_0')
l.toggleMask(Logger.TYPE)
l.info('test_info_1')
l.toggleMask(Logger.TYPE | Logger.TIME)
l.info('test_info_2')
dt2 = datetime.now()
l.toggleMask(Logger.TYPE)
l.info('test_info_3')
dt3 = datetime.now()
l.toggleMask(Logger.INFO)
# this won't get logged
l.info('test_info_4')
dt4 = datetime.now()
self.assertEqual(self.mockLog[0], 'test_info_0')
self.assertEqual(self.mockLog[1], 'INFO : test_info_1')
self.assertIn('test_info_2', self.mockLog[2])
self.assertIn(str(dt2)[:-5], self.mockLog[2])
self.assertIn('test_info_3', self.mockLog[3])
self.assertIn(str(dt3)[:-5], self.mockLog[3])
# length 4 only as last not logged
self.assertEqual(len(self.mockLog), 4)
def sourceData(log: Logger, target: str, currentSeason: bool):
'''
Obtain historical match data
:param log: a logging object
:param target: the name of match data source
:param currentSeason: True if only interested in the current season's data
'''
log.info('Downloading data from source: {}'.format(target))
config = getSweeperConfig()
dbName = config['dbName']
log.debug('Opening database: {}'.format(dbName))
with Database(dbName, SQLite3Impl()) as db, db.transaction() as t:
keys = {'name': target}
source = db.select(Source.createAdhoc(keys))
if source:
source = source[0]
else:
sys.exit('{} source not in database'.format(target))
log.debug('{}'.format(source))
sourceId = source.getId()
keys = {'source_id': sourceId, 'active': 1}
if currentSeason:
seasonMap = db.select(
Source_Season_Map.createAdhoc(keys, ('>season', )))[0:1]
else:
seasonMap = db.select(
Source_Season_Map.createAdhoc(keys, ('>season', )))
log.debug('{}'.format(seasonMap))
keys = {'source_id': sourceId}
leagueMap = db.select(Source_League_Map.createAdhoc(keys))
log.debug('{}'.format(leagueMap))
teams = db.select(Team())
log.debug('{}'.format(teams))
# Process the historical data...
for l in leagueMap:
for s in seasonMap:
url = s.getData_Url().format(l.getLeague())
try:
processMatchData(log, db, url, sourceId, s.getSeason())
except Exception as e:
log.info('Cannot process %s' % url)
# Get the upcoming fixtures too...
processMatchData(log, db, source.getUrl(), sourceId,
seasonMap[0].getSeason(), leagueMap)
def genFormTable(log:Logger, league:str, season:str, date:str=None, \
show:bool=False):
'''
Generate a form table for the subject league and season
:param log: a logging object
:param league: the subject league
:param season: the subject season
:param date: the date string up to which to generate the league YYYY-MM-DD
:param show: displays any tables as HTML when True
:returns: the league and form tables
'''
log.info('Generating form table for ' \
'league <{}> and season <{}>'.format(league, season))
config = getSweeperConfig()
dbName = config['dbName']
log.debug('Opening database: {}'.format(dbName))
with Database(dbName, SQLite3Impl()) as db, db.transaction() as t:
try:
league = db.select(League(league))[0]
except:
log.critical('No league matching the provided mnemonic exists')
sys.exit(3)
try:
season = db.select(Season(season))[0]
except:
log.critical('No season matching the provided string exists')
sys.exit(5)
headers = ['Team', 'P', 'W', 'D', 'L', 'F', 'A', 'GD', 'PTS']
schema = ['{:<20}', '{:>3}', '{:>3}', '{:>3}', '{:>3}', '{:>3}', \
'{:>3}', '{:>4}', '{:>4}']
formTable = Table(headers=headers, schema=schema, \
title='{} Form Table'.format(league.getDesc()))
leagueTable = genLeagueTable(log, league.getMnemonic(), \
season.getName(), date, show)
for team in leagueTable.getColumns()[0]:
form, results = genForm(log, team, date)
formTable.append(form.getRows())
formTable.setRows([row for row in \
sorted(sorted(formTable.getRows(), key=itemgetter(0)), \
key=lambda x : (x[8], x[7], x[5]), reverse=True)])
log.info(formTable)
if show: formTable.asHTML(show)
return leagueTable, formTable
def testPredictions(log:Logger, algoId:int, season:str, predictions:Table, \
show:bool=False):
'''
From the provided table of predictions, generate the actual betting
outcome and display.
:param log: a logging object
:param algo: the algo under test
:param season: the season under test
:param predictions: a Table of match predictions with analytics
:param show: displays any tables as HTML when True
'''
log.info('Running testPredictions for algo <{}> and season <{}>'.format( \
algoId, season))
config = getSweeperConfig()
dbName = config['dbName']
log.debug('Opening database: {}'.format(dbName))
headers = ['Date', 'Match', 'Mark', 'RO', 'AO', 'Res', 'Stk', \
'Win', 'Pot', 'Yld']
schema = ['{:<12}', '{:<40}', '{:>4}', '{:>5.2f}', '{:>5.2f}', \
'{:>3}', '{:>5}', '{:>5.2f}', '{:>5.2f}', '{:>5.2f}']
startPot = 20.0
winnings = Table(headers=headers, schema=schema,
title='{}, starting pot {} units'.format( \
predictions.getTitle(), startPot))
pot = startPot
with Database(dbName, SQLite3Impl()) as db, db.transaction() as t:
try:
algo = db.select(Algo(algoId))[0]
algo = AlgoFactory.create(algo.getName())
except Exception as e:
log.critical('No algo matching the provided id exists')
log.critical('Because...%s' % e)
sys.exit(1)
try:
season = db.select(Season(season))[0]
except Exception as e:
log.critical('No season matching the provided season exists')
log.critical('Because...%s' % e)
sys.exit(2)
# Get the match result and best odds for each prediction and then
# calculate winnings
for p in predictions.getRows():
dt = p[predictions.getHeaders().index('Date')]
fix = p[predictions.getHeaders().index('Match')]
ht, at = fix.split(' (vs) ')
m = p[predictions.getHeaders().index('Mark')]
ro = p[predictions.getHeaders().index('HO')]
keys = {'date': dt, 'home_team': ht, 'away_team': at}
match = db.select(Match.createAdhoc(keys))[0]
ao = match.getBest_Odds_H()
r = match.getResult()
keys.update({'<date': keys.pop('date')})
keys.update({'>date': season.getL_Bnd_Date()})
del keys['home_team']
del keys['away_team']
priorMatches = db.select(Match.createAdhoc(keys))
priorHTMatches = len([m for m in priorMatches \
if m.getHome_Team() == match.getHome_Team() \
or m.getAway_Team() == match.getHome_Team()])
priorATMatches = len([m for m in priorMatches \
if m.getHome_Team() == match.getAway_Team() \
or m.getAway_Team() == match.getAway_Team()])
incl = not (priorHTMatches < algo.numMatches or \
priorATMatches < algo.numMatches)
colour = None
row = [dt, fix, m, ro, ao, r]
if incl and ao >= ro:
if r == 'H':
pot += ao - 1
row += [1, ao]
colour = Table.Palette.GREEN
else:
pot -= 1
row += [1, 0]
colour = Table.Palette.RED
else:
row += [0, 0]
row += [pot, (pot - startPot) / startPot * 100.0]
if colour:
winnings.addHighlight(col='Match', pattern=fix, \
wholeRow=True, repeat=False, colour=colour)
if incl:
winnings.append([row])
log.info(winnings)
if show:
winnings.asHTML(show)
return winnings
def analyseStatistics(
log:Logger, algoId:int, league:str=None, lbnd:int=None, ubnd:int=None, \
backtest:bool=False):
'''
Analyse statistics for the algo and league combination
:param log: a logging object
:param algoId: the algo subject
:param league: the league subject, all if unset
:param lbnd: include marks above this value
:param ubnd: include marks below this value
:param backtest: run in backtest mode
'''
log.info('Analysing statistics for league <{}> with algo <{}> and ' \
'backtest <{}>'.format(league if league else 'ALL', algoId, \
backtest))
config = getSweeperConfig()
dbName = config['dbName']
log.debug('Opening database: {}'.format(dbName))
with Database(dbName, SQLite3Impl()) as db, db.transaction() as t:
try:
# In backtest mode use the inverse algoId to retrieve config,
# ratings and stats:
if backtest: algoId = -algoId
keys = {'algo_id': algoId}
order = ['>generation_date']
if league:
keys.update({'league': league})
order.append('>league')
if lbnd and ubnd: keys.update({'>mark': lbnd, '<mark': ubnd})
statistics = db.select(Statistics.createAdhoc(keys, order))
if not statistics: raise Exception('No statistics')
lastGenDate = statistics[0].getGeneration_Date()
statistics = [s for s in statistics \
if s.getGeneration_Date() == lastGenDate]
except:
log.critical('No statistics matching the provided algo and ' \
'league exists')
sys.exit(2)
for league, group in itertools.groupby(\
statistics, lambda x : x.getLeague()):
statsGrp = list(group)
x = [s.getMark() for s in statsGrp if s.getMark_Freq() > 0]
hY = [s.getHome_Freq() / s.getMark_Freq() * 100 \
for s in statsGrp if s.getMark_Freq() > 0]
hF = [s.getHome_Freq() for s in statsGrp if s.getMark_Freq() > 0]
dY = [s.getDraw_Freq() / s.getMark_Freq() * 100 \
for s in statsGrp if s.getMark_Freq() > 0]
dF = [s.getDraw_Freq() for s in statsGrp if s.getMark_Freq() > 0]
aY = [s.getAway_Freq() / s.getMark_Freq() * 100 \
for s in statsGrp if s.getMark_Freq() > 0]
aF = [s.getAway_Freq() for s in statsGrp if s.getMark_Freq() > 0]
slope, intercept, r, p, stderr = stats.linregress(x, hY)
r2 = r**2
log.info('{:>4} Home: {:>4.2f} {:>4.2f} {:>4.2} {:>4.2f} ' \
'{:>4.2f} {:>4.2}'.format(league, slope, intercept, p, \
r, r2, stderr))
createPlot(x, hY, hF, intercept, slope, league + ' home')
slope, intercept, r, p, stderr = stats.linregress(x, dY)
r2 = r**2
log.info('{:>4} Draw: {:>4.2f} {:>4.2f} {:>4.2} {:>4.2f} ' \
'{:>4.2f} {:>4.2}'.format(league, slope, intercept, p, \
r, r2, stderr))
createPlot(x, dY, dF, intercept, slope, league + ' draw')
slope, intercept, r, p, stderr = stats.linregress(x, aY)
r2 = r**2
log.info('{:>4} Away: {:>4.2f} {:>4.2f} {:>4.2} {:>4.2f} ' \
'{:>4.2f} {:>4.2}'.format(league, slope, intercept, p, \
r, r2, stderr))
createPlot(x, aY, aF, intercept, slope, league + ' away')
def genStats(log: Logger,
algoId: int,
league: str = None,
backtest: bool = False):
'''
Generate statistics on the marked matches
:param log: a logging object
:param algoId: the algo to apply
:param league: the league to apply the algo over, all if unset
:param backtest: run in backtest mode
'''
log.info('Generating statistics for league <{}> with algo <{}> and ' \
'backtest <{}>'.format(league if league else 'ALL', algoId, \
backtest))
config = getSweeperConfig()
dbName = config['dbName']
log.debug('Opening database: {}'.format(dbName))
with Database(dbName, SQLite3Impl()) as db, db.transaction() as t:
try:
algo = db.select(Algo(algoId))[0]
algo = AlgoFactory.create(algo.getName())
# In backtest mode use the inverse algoId to retrieve config,
# ratings and stats:
if backtest: algoId = -algoId
except:
log.critical('No algo matching the provided id exists')
sys.exit(3)
try:
if league:
leagues = db.select(League(league))
else:
leagues = db.select(League())
except:
log.critical('No league matching the provided mnemonic exists')
sys.exit(4)
for league in leagues:
stats = {}
def getStatisticsForResult(result, setfn, getfn):
keys = {'league': league.getMnemonic(), 'result': result}
order = ['>date']
for m in db.select(Match.createAdhoc(keys, order)):
rating = db.select(
Rating(m.getDate(), m.getLeague(), m.getHome_Team(),
m.getAway_Team(), algoId))
if rating:
mark = rating[0].getMark()
s = stats.get(mark, Statistics(str( \
datetime.now().date()), algoId, \
league.getMnemonic(), mark, 0, 0, 0, 0))
s.setMark_Freq(s.getMark_Freq() + 1)
setfn(s, getfn(s) + 1)
stats[mark] = s
getStatisticsForResult('H', Statistics.setHome_Freq,
Statistics.getHome_Freq)
getStatisticsForResult('D', Statistics.setDraw_Freq,
Statistics.getDraw_Freq)
getStatisticsForResult('A', Statistics.setAway_Freq,
Statistics.getAway_Freq)
for k, v in stats.items():
db.upsert(v)
def genForm(log: Logger, team: str, date: str = None, show: bool = False):
'''
Generate form over the previous 6 matches for the team provided
:param log: a logging object
:param team: the subject team
:param date: search date, today if None
:param show: displays any tables as HTML when True
'''
log.info('Generating form for date <{}> and team <{}>'.format(date, team))
config = getSweeperConfig()
dbName = config['dbName']
log.debug('Opening database: {}'.format(dbName))
with Database(dbName, SQLite3Impl()) as db, db.transaction() as t:
if date:
dt = datetime.strptime(date, '%Y-%m-%d')
else:
dt = datetime.today().date()
dt = dt + timedelta(days=1)
try:
keys = {'<date' : dt.strftime('%Y-%m-%d'), 'home_team' : team, \
'!result' : ''}
order = {'>date'}
matches1 = [m for m in db.select(Match.createAdhoc(keys, order))]
del keys['home_team']
keys['away_team'] = team
matches2 = [m for m in db.select(Match.createAdhoc(keys, order))]
except:
log.critical("Couldn't find matches for team and date provided")
sys.exit(2)
matches = sorted(matches1 + matches2,
key=lambda m: m.getDate(),
reverse=True)[0:6]
form = Form()
for m in matches:
form.played += 1
if m.getHome_Team() == team:
form.glfor += m.getHome_Goals()
form.glagn += m.getAway_Goals()
else:
form.glfor += m.getAway_Goals()
form.glagn += m.getHome_Goals()
form.gldif = form.glfor - form.glagn
if m.getResult() == 'H':
if m.getHome_Team() == team:
form.won += 1
form.points += 3
else:
form.lost += 1
elif m.getResult() == 'D':
form.drawn += 1
form.points += 1
elif m.getResult() == 'A':
if m.getAway_Team() == team:
form.won += 1
form.points += 3
else:
form.lost += 1
else:
raise Exception("Wasn't expecting that!")
headers = ['Team', 'P', 'W', 'D', 'L', 'F', 'A', 'GD', 'PTS']
schema = ['{:<20}', '{:>3}', '{:>3}', '{:>3}', '{:>3}', '{:>3}', \
'{:>3}', '{:>4}', '{:>4}']
t1 = Table(headers=headers, schema=schema)
t1.append([[team, *form.asList()]])
log.info(t1)
if show: t1.asHTML(show)
headers = ['Date', 'Home Team', 'HTG', 'ATG', 'Away Team']
schema = ['{:<12}', '{:<20}', '{:>3}', '{:>3}', '{:>20}']
t2 = Table(headers=headers, schema=schema)
t2.append([[m.getDate(), m.getHome_Team(), m.getHome_Goals(), \
m.getAway_Goals(), m.getAway_Team()] for m in matches])
t2.addHighlight('Home Team', team, False)
log.info(t2)
if show: t2.asHTML(show)
return t1, t2
def presentFixtures(log:Logger, algoId:int, league:str=None, show:bool=False, \
mail:bool=False, backtest:bool=False, season:str=None):
'''
Present the latest set of fixtures with all the appropriate ratings.
:param log: a logging object
:param league: the subject league, None signifies all available leagues
:param show: displays any tables as HTML when True
:param mail: send as email
:param backtest: run in backtest mode
:param season: season to run backtest for
'''
log.info('Presenting fixtures for algo <{}>, league <{}> and backtest ' \
'<{}> for season <{}>'.format(algoId, league if league else 'ALL', \
backtest, season))
config = getSweeperConfig()
dbName = config['dbName']
log.debug('Opening database: {}'.format(dbName))
with Database(dbName, SQLite3Impl()) as db, db.transaction() as t:
date = (datetime.today() - timedelta(days=1)).strftime('%Y-%m-%d')
try:
algo = db.select(Algo(algoId))[0]
algo = AlgoFactory.create(algo.getName())
except Exception as e:
log.critical('No algo matching the provided id exists')
log.critical('Because...%s' % e)
sys.exit(2)
if backtest:
# In backtest mode use the inverse algoId to retrieve config,
# ratings and stats and process all matches irrespective of
# existing results
algoId = -algoId
if season:
try:
season = db.select(Season(season))[0]
except Exception as e:
log.critical( \
'No season matching the provided season exists')
sys.exit(3)
else:
log.critical('Must specify season with backtest')
sys.exit(4)
keys = {'>date' : season.getL_Bnd_Date(), \
'<date' : season.getU_Bnd_Date()}
else:
keys = {'>date': date, 'result': ''}
try:
if league: keys.update({'league': league})
order = ['<league', '<date']
fixtures = db.select(Match.createAdhoc(keys, order))
if not fixtures: raise Exception('No fixtures')
except Exception as e:
log.critical("Couldn't find fixtures for league and date " \
"provided, run sourcedata?")
log.critical('Because...{}'.format(e))
sys.exit(5)
try:
if 'result' in keys: del keys['result']
if '<date' in keys: keys.update({'<match_date': keys.pop('<date')})
del keys['>date']
dt = datetime.strptime(min(f.getDate() for f in fixtures), \
'%Y-%m-%d') - timedelta(days=1)
keys.update({'>match_date': dt.strftime('%Y-%m-%d')})
keys.update({'algo_id': algoId})
order = ['<league', '<match_date']
ratings = db.select(Rating.createAdhoc(keys, order))
log.debug('Num fixtures {}, ratings {}'.format(len(fixtures), \
len(ratings)))
if len(fixtures) != len(ratings):
raise Exception('Mismatched ratings')
except Exception as e:
log.critical("Couldn't find algo ratings for all fixtures, " \
"run analysematches?")
log.critical('Because...{}'.format(e))
sys.exit(6)
try:
del keys['>match_date']
if '<match_date' in keys: del keys['<match_date']
keys.update({'>generation_date': date})
order = ['>generation_date']
stats = db.select(Statistics.createAdhoc(keys, order))
if not stats: raise Exception('No statistics')
lastGenDate = stats[0].getGeneration_Date()
stats = [s for s in stats if s.getGeneration_Date() == lastGenDate]
except Exception as e:
log.critical("Couldn't find algo statistics for league and date, " \
"run genstats?")
log.critical('Because...{}'.format(e))
sys.exit(7)
def statsSummary(s: Statistics):
if s.getMark() == 99:
return 0, 0, 0.0, 0.0, 0, 0.0, 0.0, 0, 0.0, 0.0
markF = s.getMark_Freq()
homeF = s.getHome_Freq()
homeP = (homeF / markF) * 100.0 if markF else 0.0
homeO = 100.0 / homeP if homeP else 99.99
drawF = s.getDraw_Freq()
drawP = (drawF / markF) * 100.0 if markF else 0.0
drawO = 100.0 / drawP if drawP else 99.99
awayF = s.getAway_Freq()
awayP = (awayF / markF) * 100.0 if markF else 0.0
awayO = 100.0 / awayP if awayP else 99.99
return markF, homeF, homeP, homeO, drawF, drawP, drawO, awayF, \
awayP, awayO
for r in itertools.filterfalse(lambda r : r.getMark() in \
[s.getMark() for s in stats if r.getLeague() == s.getLeague()],\
ratings):
stats.append(Statistics(r.getMatch_Date(), r.getAlgo_Id(), \
r.getLeague(), r.getMark(), 0, 0, 0, 0))
analytics = map(lambda r : [(r, statsSummary(s)) for s in stats \
if r.getMark() == s.getMark() \
and r.getLeague() == s.getLeague()], ratings)
presentation = zip(fixtures, analytics)
tables = {}
mailText = 'Visit the website for more details - http://www.sweeperfootball.com<br/><br/>'
for i, (league, group) in enumerate(itertools.groupby(presentation, \
lambda x : x[0].getLeague())):
try:
leagueDesc = db.select(League(league))[0].getDesc()
except Exception as e:
log.critical("Couldn't find league")
log.critical('Because..{}'.format(e))
sys.exit(5)
try:
keys = {'league': league, 'algo_id': algoId}
order = ['>config_date']
algoCfg = db.select(Algo_Config.createAdhoc(keys, order))[0]
except Exception as e:
log.critical("Couldn't find algo config for league")
log.critical('Because...{}'.format(e))
sys.exit(6)
presGrp = list(group)
headers = ['Date', 'Match', 'Mark', 'M#', \
'H#', 'H%', 'HO', 'D#', 'D%', 'DO', 'A#', 'A%', 'AO']
schema = ['{:<12}', '{:<40}', '{:>4}', '{:>4}', \
'{:>4}', '{:>5.2f}', '{:>5.2f}', '{:>4}', '{:>5.2f}', \
'{:>5.2f}', '{:>4}', '{:>5.2f}', '{:>5.2f}']
t = Table(headers=headers, schema=schema, \
title='{} Fixtures'.format(leagueDesc))
if backtest:
# if we are backtesting then only return the predictions
t.append([[f.getDate(), '{} (vs) {}'.format(f.getHome_Team(), \
f.getAway_Team()), r.getMark(), *a] \
for f, [(r, a)] in presGrp \
if r.getMark() > algoCfg.getL_Bnd_Mark() \
and r.getMark() < algoCfg.getU_Bnd_Mark()])
return t
t.append([[f.getDate(), '{} (vs) {}'.format(f.getHome_Team(), \
f.getAway_Team()), r.getMark(), *a] \
for f, [(r, a)] in presGrp])
t.setHighlights([['Match', '{} (vs) {}'.format(f.getHome_Team(), \
f.getAway_Team()), False, False] \
for f, [(r, a)] in presGrp \
if r.getMark() > algoCfg.getL_Bnd_Mark() \
and r.getMark() < algoCfg.getU_Bnd_Mark()])
t.htmlReplacements([['(vs)', '<br/>']])
try:
keys = {'>u_bnd_date': date, '<l_bnd_date': date}
season = db.select(Season.createAdhoc(keys))[0].getName()
except Exception as e:
log.critical("Couldn't find season for date")
log.critical('Because...{}'.format(e))
sys.exit(6)
mask = log.getMask()
log.setMask(mask & ~Logger.INFO)
leagueTable, formTable = genFormTable(log, league, season, date)
log.setMask(mask)
log.info(t)
log.info(formTable)
tables[leagueDesc] = (t, leagueTable, formTable)
if show:
t.asHTML(show)
formTable.asHTML(show)
if mail:
if not i:
mailText += t.asHTML().replace('</body>', '') + '<br/>'
else:
mailText += t.asHTML(fullyFormed=False) + '<br/>'
mailText += formTable.asHTML(fullyFormed=False) + '<br/>'
if mail:
mailText = 'MIME-Version: 1.0\nContent-type: text/html\nSubject: Sweeper Football Predictions\n\n{}</body>'.format(
mailText)
#mailText = 'MIME-Version: 1.0\nContent-type: text/html\nSubject: Sweeper Football Predictions - PREDICTIONS AVAILABLE FROM THIS WEEK!\n\n{}</body>'.format(mailText)
mailCfg = getSweeperConfig('mail.cfg')
fromAddr = mailCfg['fromAddr']
subs = db.select(Subscriber.createAdhoc({'include': 1}))
toAddrs = [s.getEmail() for s in subs]
server = smtplib.SMTP(mailCfg['svr'], int(mailCfg['port']))
server.ehlo()
server.starttls()
server.ehlo()
server.login(fromAddr, mailCfg['pwd'])
server.sendmail(fromAddr, toAddrs, mailText)
server.quit()
log.info('email sent to: {!s}'.format(toAddrs))
return tables
def runSweeper(log:Logger, algoId:int, league:str=None, season:str=None, \
show:bool=False, mail:bool=False, backtest:bool=False):
'''
Run the entire sweeper suite
:param log: a logging object
:param league: the subject league, None signifies all available leagues
:param season: the subject season, None current season
:param show: displays any tables as HTML when True
:param mail: send as email
:param backtest: run in backtest mode
'''
log.info('Running sweeper suite for algo <{}>, league <{}>, season <{}>, '\
'backtest <{}>'.format(algoId, league if league else 'ALL', \
season if season else 'CURRENT', backtest))
log.info('\nSourcing data...\n')
sourceData(log=log, target='Football-Data', currentSeason=not season)
log.info('\nSourcing data...Done\n')
log.info('\nAnalysing matches...\n')
analyseMatches(log=log, algoId=algoId, league=league, season=season, \
backtest=backtest)
log.info('\nAnalysing matches...Done\n')
log.info('\nGenerating stats...\n')
genStats(log=log, algoId=algoId, league=league, backtest=backtest)
log.info('\nGenerating stats...Done\n')
if not backtest:
log.info('\nCreating web page...\n')
dt = datetime.today().strftime('%Y-%m-%d')
createWebPage(log=log, algoId=algoId, date=dt, league=league, show=show)
log.info('\nCreating web page...Done\n')
log.info('\nPresent fixtures...\n')
tables = presentFixtures(log=log, algoId=algoId, league=league, \
show=show, season=season, mail=mail, backtest=backtest)
log.info('\nPresent fixtures...Done\n')
if backtest:
log.info('\nTesting fixtures...\n')
winnings = testPredictions(log=log, algoId=algoId, season=season, \
predictions=tables, show=show)
log.info('\nTesting fixtures...Done\n')
return winnings
return tables
def analyseMatches(log:Logger, algoId:int, league:str=None, season:str=None, \
backtest:bool=False):
'''
Mark all unmarked matches
:param log: a logging object
:param algoId: the algo to apply
:param league: the league to apply the algo over, None means ALL
:param season: the season to apply the algo over, None means ALL
:param backtest: run in backtest mode
'''
log.info('Analysing matches for league <{}>, season <{}> with algo <{}> ' \
'and backtest <{}>'.format(league if league else 'ALL', \
season if season else 'ALL', algoId, backtest))
config = getSweeperConfig()
dbName = config['dbName']
log.debug('Opening database: {}'.format(dbName))
with Database(dbName, SQLite3Impl()) as db, db.transaction() as t:
try:
algo = db.select(Algo(algoId))[0]
algo = AlgoFactory.create(algo.getName())
# In backtest mode use the inverse algoId to retrieve config,
# ratings and stats
if backtest: algoId = -algoId
except:
log.critical('No algo matching the provided id exists')
sys.exit(2)
try:
if league:
leagues = db.select(League(league))
else:
leagues = db.select(League())
except:
log.critical('No league matching the provided mnemonic exists')
sys.exit(3)
try:
if season: season = db.select(Season(season))[0]
except:
log.critical('No season matching the provided season exists')
sys.exit(4)
keys = {'algo_id': algoId, '<mark': 99}
ratings = db.select(Rating.createAdhoc(keys))
ratedMatchKeys = [MatchKeys(r.getMatch_Date(), r.getLeague(), \
r.getHome_Team(), r.getAway_Team()) for r in ratings]
log.info('Found {} ratings for algo {}'.format(len(ratedMatchKeys), \
algoId))
for league in leagues:
if season:
keys = {'league' : league.getMnemonic(), \
'>date' : season.getL_Bnd_Date(), \
'<date' : season.getU_Bnd_Date()}
else:
keys = {'league': league.getMnemonic()}
order = ['>league', '>date']
matches = db.select(Match.createAdhoc(keys, order))
unmarked = list(filter(lambda x : x._keys not in ratedMatchKeys, \
matches))
results = list(filter(lambda x: x.getResult() != '', matches))
log.info('{} {} matches found unmarked'.format(len(unmarked), \
league.getMnemonic()))
for m in unmarked:
hTeamMatches = list(filter(lambda x : m.getHome_Team() in \
(x.getHome_Team(), x.getAway_Team()) and x.getDate() \
< m.getDate(), results))
aTeamMatches = list(filter(lambda x : m.getAway_Team() in \
(x.getHome_Team(), x.getAway_Team()) and x.getDate() \
< m.getDate(), results))
mark = algo.markMatch(m, hTeamMatches, aTeamMatches)
if mark is not None:
db.upsert(Rating(m.getDate(), m.getLeague(), \
m.getHome_Team(), m.getAway_Team(), algoId, mark))
def genLeagueTable(log:Logger, league:str, season:str, date:str=None, \
show:bool=False):
'''
Generate a league table for the subject league and season
:param log: a logging object
:param league: the subject league
:param season: the subject season
:param date: the date string up to which to generate the league YYYY-MM-DD
:param show: displays any tables as HTML when True
'''
log.info('Generating league table for ' \
'league <{}> and season <{}>'.format(league, season))
config = getSweeperConfig()
dbName = config['dbName']
log.debug('Opening database: {}'.format(dbName))
with Database(dbName, SQLite3Impl()) as db, db.transaction() as t:
try:
league = db.select(League(league))[0]
except:
log.critical('No league matching the provided mnemonic exists')
sys.exit(3)
try:
season = db.select(Season(season))[0]
except:
log.critical('No season matching the provided string exists')
sys.exit(4)
ubnd = date if date is not None else season.getU_Bnd_Date()
keys = {'league' : league.getMnemonic(), '!result' : '', '>date' : \
season.getL_Bnd_Date(), '<date' : ubnd}
matches = [m for m in db.select(Match.createAdhoc(keys))]
log.info('{} {} matches found'.format(len(matches), \
league.getMnemonic()))
keys = {'league' : league.getMnemonic(), 'season' : season.getName()}
teams = db.select(Team.createAdhoc(keys))
table = dict([(t.getName(), Form()) for t in teams])
for m in matches:
table[m.getHome_Team()] = h = table.get(m.getHome_Team(), Form())
table[m.getAway_Team()] = a = table.get(m.getAway_Team(), Form())
h.played += 1
a.played += 1
h.glfor += m.getHome_Goals()
h.glagn += m.getAway_Goals()
h.gldif = h.glfor - h.glagn
a.glfor += m.getAway_Goals()
a.glagn += m.getHome_Goals()
a.gldif = a.glfor - a.glagn
if m.getResult() == 'H':
h.won += 1
h.points += 3
a.lost += 1
elif m.getResult() == 'D':
h.drawn += 1
h.points += 1
a.drawn += 1
a.points += 1
elif m.getResult() == 'A':
a.won += 1
a.points += 3
h.lost += 1
else:
raise Exception("Empty result, wasn't expecting that")
headers = ['Team', 'P', 'W', 'D', 'L', 'F', 'A', 'GD', 'PTS']
schema = ['{:<20}', '{:>3}', '{:>3}', '{:>3}', '{:>3}', '{:>3}', \
'{:>3}', '{:>4}', '{:>4}']
t = Table(headers=headers, schema=schema, title='{} Table'.format(\
league.getDesc()))
t.append([[row[0], *row[1].asList()] for row in \
sorted(sorted(table.items(), key=itemgetter(0)), \
key=lambda x : (x[1].points, x[1].gldif, x[1].glfor), \
reverse=True)])
log.info(t)
if show: t.asHTML(show)
return t