def test_insert(self): # Disable Foreign Keys checks for this test TestStatistics.db.disableForeignKeys() with TestStatistics.db.transaction() as t: TestStatistics.db.upsert( Statistics('statistics generation_date TD INS', 100, 'league mnemonic TD INS', 100, 100, 100, 100, 100)) objs = TestStatistics.db.select(Statistics()) self.assertEqual(len(objs), 3) d = eval( "{'generation_date': 'statistics generation_date TD INS', 'algo_id': 100, 'league': 'league mnemonic TD INS', 'mark': 100}" ) for k, v in d.items(): self.assertEqual(objs[2].__getattribute__('get' + k.title())(), v) d = eval( "{'mark_freq': 100, 'home_freq': 100, 'away_freq': 100, 'draw_freq': 100}" ) for k, v in d.items(): self.assertEqual(objs[2].__getattribute__('get' + k.title())(), v) # force a rollback t.fail()
def test_select(self): objs = TestStatistics.db.select(Statistics()) self.assertEqual(len(objs), 2) self.assertEqual(objs[0].getGeneration_Date(), 'statistics generation_date TD') self.assertEqual(objs[0].getAlgo_Id(), 98) self.assertEqual(objs[0].getLeague(), 'league mnemonic TD') self.assertEqual(objs[0].getMark(), 98) self.assertEqual(objs[0].getMark_Freq(), 98) self.assertEqual(objs[0].getHome_Freq(), 98) self.assertEqual(objs[0].getAway_Freq(), 98) self.assertEqual(objs[0].getDraw_Freq(), 98) self.assertEqual(objs[1].getGeneration_Date(), 'statistics generation_date TD2') self.assertEqual(objs[1].getAlgo_Id(), 99) self.assertEqual(objs[1].getLeague(), 'league mnemonic TD2') self.assertEqual(objs[1].getMark(), 99) self.assertEqual(objs[1].getMark_Freq(), 99) self.assertEqual(objs[1].getHome_Freq(), 99) self.assertEqual(objs[1].getAway_Freq(), 99) self.assertEqual(objs[1].getDraw_Freq(), 99) objs = TestStatistics.db.select( Statistics('statistics generation_date TD', 98, 'league mnemonic TD', 98)) self.assertEqual(len(objs), 1) self.assertEqual(objs[0].getGeneration_Date(), 'statistics generation_date TD') self.assertEqual(objs[0].getAlgo_Id(), 98) self.assertEqual(objs[0].getLeague(), 'league mnemonic TD') self.assertEqual(objs[0].getMark(), 98) self.assertEqual(objs[0].getMark_Freq(), 98) self.assertEqual(objs[0].getHome_Freq(), 98) self.assertEqual(objs[0].getAway_Freq(), 98) self.assertEqual(objs[0].getDraw_Freq(), 98) objs = TestStatistics.db.select( Statistics.createAdhoc({ 'mark_freq': 98, 'home_freq': 98, 'away_freq': 98, 'draw_freq': 98 })) self.assertEqual(len(objs), 1) self.assertEqual(objs[0].getGeneration_Date(), 'statistics generation_date TD') self.assertEqual(objs[0].getAlgo_Id(), 98) self.assertEqual(objs[0].getLeague(), 'league mnemonic TD') self.assertEqual(objs[0].getMark(), 98) self.assertEqual(objs[0].getMark_Freq(), 98) self.assertEqual(objs[0].getHome_Freq(), 98) self.assertEqual(objs[0].getAway_Freq(), 98) self.assertEqual(objs[0].getDraw_Freq(), 98)
def test_update(self): # Disable Foreign Keys checks for this test TestStatistics.db.disableForeignKeys() with TestStatistics.db.transaction() as t: TestStatistics.db.upsert( Statistics('statistics generation_date TD', 98, 'league mnemonic TD', 98, 100, 100, 100, 100)) objs = TestStatistics.db.select( Statistics('statistics generation_date TD', 98, 'league mnemonic TD', 98)) self.assertEqual(len(objs), 1) self.assertEqual(objs[0].getGeneration_Date(), 'statistics generation_date TD') self.assertEqual(objs[0].getAlgo_Id(), 98) self.assertEqual(objs[0].getLeague(), 'league mnemonic TD') self.assertEqual(objs[0].getMark(), 98) d = eval( "{'mark_freq': 100, 'home_freq': 100, 'away_freq': 100, 'draw_freq': 100}" ) for k, v in d.items(): self.assertEqual(objs[0].__getattribute__('get' + k.title())(), v) # force a rollback t.fail() with TestStatistics.db.transaction() as t: statistics = TestStatistics.db.select( Statistics('statistics generation_date TD', 98, 'league mnemonic TD', 98))[0] for k, v in d.items(): statistics.__getattribute__('set' + k.title())(v) TestStatistics.db.upsert(statistics) objs = TestStatistics.db.select( Statistics('statistics generation_date TD', 98, 'league mnemonic TD', 98)) self.assertEqual(len(objs), 1) self.assertEqual(objs[0].getGeneration_Date(), 'statistics generation_date TD') self.assertEqual(objs[0].getAlgo_Id(), 98) self.assertEqual(objs[0].getLeague(), 'league mnemonic TD') self.assertEqual(objs[0].getMark(), 98) for k, v in d.items(): self.assertEqual(objs[0].__getattribute__('get' + k.title())(), v) # force a rollback t.fail()
def test_delete(self): # Disable Foreign Keys checks for this test TestStatistics.db.disableForeignKeys() with TestStatistics.db.transaction() as t: TestStatistics.db.delete( Statistics('statistics generation_date TD', 98, 'league mnemonic TD', 98)) objs = TestStatistics.db.select(Statistics()) self.assertEqual(len(objs), 1) # force a rollback t.fail()
def statsSummary(s: Statistics): if s.getMark() == 99: return 0, 0, 0.0, 0.0, 0, 0.0, 0.0, 0, 0.0, 0.0 markF = s.getMark_Freq() homeF = s.getHome_Freq() homeP = (homeF / markF) * 100.0 if markF else 0.0 homeO = 100.0 / homeP if homeP else 99.99 drawF = s.getDraw_Freq() drawP = (drawF / markF) * 100.0 if markF else 0.0 drawO = 100.0 / drawP if drawP else 99.99 awayF = s.getAway_Freq() awayP = (awayF / markF) * 100.0 if markF else 0.0 awayO = 100.0 / awayP if awayP else 99.99 return markF, homeF, homeP, homeO, drawF, drawP, drawO, awayF, \ awayP, awayO
def test_repr(self): obj = Statistics('statistics generation_date TD', 98, 'league mnemonic TD', 98, 98, 98, 98, 98) self.assertEqual( str(obj), "statistics : Keys {'generation_date': 'statistics generation_date TD', 'algo_id': 98, 'league': 'league mnemonic TD', 'mark': 98} : Values {'mark_freq': 98, 'home_freq': 98, 'away_freq': 98, 'draw_freq': 98}" )
def test_create(self): obj = Statistics.create(('statistics generation_date TD', 98, 'league mnemonic TD', 98, 98, 98, 98, 98)) self.assertEqual(obj.getGeneration_Date(), 'statistics generation_date TD') self.assertEqual(obj.getAlgo_Id(), 98) self.assertEqual(obj.getLeague(), 'league mnemonic TD') self.assertEqual(obj.getMark(), 98) self.assertEqual(obj.getMark_Freq(), 98) self.assertEqual(obj.getHome_Freq(), 98) self.assertEqual(obj.getAway_Freq(), 98) self.assertEqual(obj.getDraw_Freq(), 98)
def getStatisticsForResult(result, setfn, getfn): keys = {'league': league.getMnemonic(), 'result': result} order = ['>date'] for m in db.select(Match.createAdhoc(keys, order)): rating = db.select( Rating(m.getDate(), m.getLeague(), m.getHome_Team(), m.getAway_Team(), algoId)) if rating: mark = rating[0].getMark() s = stats.get(mark, Statistics(str( \ datetime.now().date()), algoId, \ league.getMnemonic(), mark, 0, 0, 0, 0)) s.setMark_Freq(s.getMark_Freq() + 1) setfn(s, getfn(s) + 1) stats[mark] = s
def test_analyseMatches(self): analyseMatches(self.mock_Logger, 1, 'league mnemonic TD') calls = ( call.info('Analysing matches for league <league ' \ 'mnemonic TD> with algo <1>'), call.debug('Opening database: ./db/test.db'), call.debug('Last rating for match 99'), call.debug('7 matches found to mark'), ) self.mock_Logger.assert_has_calls(calls) with Database(self.dbName, SQLite3Impl()) as db: stats = db.select(Statistics()) ratings = db.select(Rating()) self.assertEquals(len(stats), 2) self.assertEquals(stats[0].getGeneration_Date(), \ str(datetime.now().date())) self.assertEquals(stats[0].getAlgo_Id(), 1) self.assertEquals(stats[0].getLeague(), 'league mnemonic TD') self.assertEquals(stats[0].getMark(), -3) self.assertEquals(stats[0].getMark_Freq(), 50.0) self.assertEquals(stats[0].getHome_Freq(), 100.0) self.assertEquals(stats[0].getAway_Freq(), 0.0) self.assertEquals(stats[0].getDraw_Freq(), 0.0) self.assertEquals(stats[1].getGeneration_Date(), \ str(datetime.now().date())) self.assertEquals(stats[1].getAlgo_Id(), 1) self.assertEquals(stats[1].getLeague(), 'league mnemonic TD') self.assertEquals(stats[1].getMark(), -2) self.assertEquals(stats[1].getMark_Freq(), 50.0) self.assertEquals(stats[1].getHome_Freq(), 100.0) self.assertEquals(stats[1].getAway_Freq(), 0.0) self.assertEquals(stats[1].getDraw_Freq(), 0.0) self.assertEquals(len(ratings), 2) self.assertEquals(ratings[0].getMatch_Oid(), 6) self.assertEquals(ratings[0].getAlgo_Id(), 1) self.assertEquals(ratings[0].getMark(), -3) self.assertEquals(ratings[1].getMatch_Oid(), 7) self.assertEquals(ratings[1].getAlgo_Id(), 1) self.assertEquals(ratings[1].getMark(), -2)
def analyseStatistics( log:Logger, algoId:int, league:str=None, lbnd:int=None, ubnd:int=None, \ backtest:bool=False): ''' Analyse statistics for the algo and league combination :param log: a logging object :param algoId: the algo subject :param league: the league subject, all if unset :param lbnd: include marks above this value :param ubnd: include marks below this value :param backtest: run in backtest mode ''' log.info('Analysing statistics for league <{}> with algo <{}> and ' \ 'backtest <{}>'.format(league if league else 'ALL', algoId, \ backtest)) config = getSweeperConfig() dbName = config['dbName'] log.debug('Opening database: {}'.format(dbName)) with Database(dbName, SQLite3Impl()) as db, db.transaction() as t: try: # In backtest mode use the inverse algoId to retrieve config, # ratings and stats: if backtest: algoId = -algoId keys = {'algo_id': algoId} order = ['>generation_date'] if league: keys.update({'league': league}) order.append('>league') if lbnd and ubnd: keys.update({'>mark': lbnd, '<mark': ubnd}) statistics = db.select(Statistics.createAdhoc(keys, order)) if not statistics: raise Exception('No statistics') lastGenDate = statistics[0].getGeneration_Date() statistics = [s for s in statistics \ if s.getGeneration_Date() == lastGenDate] except: log.critical('No statistics matching the provided algo and ' \ 'league exists') sys.exit(2) for league, group in itertools.groupby(\ statistics, lambda x : x.getLeague()): statsGrp = list(group) x = [s.getMark() for s in statsGrp if s.getMark_Freq() > 0] hY = [s.getHome_Freq() / s.getMark_Freq() * 100 \ for s in statsGrp if s.getMark_Freq() > 0] hF = [s.getHome_Freq() for s in statsGrp if s.getMark_Freq() > 0] dY = [s.getDraw_Freq() / s.getMark_Freq() * 100 \ for s in statsGrp if s.getMark_Freq() > 0] dF = [s.getDraw_Freq() for s in statsGrp if s.getMark_Freq() > 0] aY = [s.getAway_Freq() / s.getMark_Freq() * 100 \ for s in statsGrp if s.getMark_Freq() > 0] aF = [s.getAway_Freq() for s in statsGrp if s.getMark_Freq() > 0] slope, intercept, r, p, stderr = stats.linregress(x, hY) r2 = r**2 log.info('{:>4} Home: {:>4.2f} {:>4.2f} {:>4.2} {:>4.2f} ' \ '{:>4.2f} {:>4.2}'.format(league, slope, intercept, p, \ r, r2, stderr)) createPlot(x, hY, hF, intercept, slope, league + ' home') slope, intercept, r, p, stderr = stats.linregress(x, dY) r2 = r**2 log.info('{:>4} Draw: {:>4.2f} {:>4.2f} {:>4.2} {:>4.2f} ' \ '{:>4.2f} {:>4.2}'.format(league, slope, intercept, p, \ r, r2, stderr)) createPlot(x, dY, dF, intercept, slope, league + ' draw') slope, intercept, r, p, stderr = stats.linregress(x, aY) r2 = r**2 log.info('{:>4} Away: {:>4.2f} {:>4.2f} {:>4.2} {:>4.2f} ' \ '{:>4.2f} {:>4.2}'.format(league, slope, intercept, p, \ r, r2, stderr)) createPlot(x, aY, aF, intercept, slope, league + ' away')
def test_keys_adhoc(self): l = Statistics.createAdhoc(None) self.assertEqual(l.getTable(), 'statistics') self.assertTrue(l._keys.getFields() is None)
def test_isNullable(self): obj = Statistics() self.assertTrue(True)
def presentFixtures(log:Logger, algoId:int, league:str=None, show:bool=False, \ mail:bool=False, backtest:bool=False, season:str=None): ''' Present the latest set of fixtures with all the appropriate ratings. :param log: a logging object :param league: the subject league, None signifies all available leagues :param show: displays any tables as HTML when True :param mail: send as email :param backtest: run in backtest mode :param season: season to run backtest for ''' log.info('Presenting fixtures for algo <{}>, league <{}> and backtest ' \ '<{}> for season <{}>'.format(algoId, league if league else 'ALL', \ backtest, season)) config = getSweeperConfig() dbName = config['dbName'] log.debug('Opening database: {}'.format(dbName)) with Database(dbName, SQLite3Impl()) as db, db.transaction() as t: date = (datetime.today() - timedelta(days=1)).strftime('%Y-%m-%d') try: algo = db.select(Algo(algoId))[0] algo = AlgoFactory.create(algo.getName()) except Exception as e: log.critical('No algo matching the provided id exists') log.critical('Because...%s' % e) sys.exit(2) if backtest: # In backtest mode use the inverse algoId to retrieve config, # ratings and stats and process all matches irrespective of # existing results algoId = -algoId if season: try: season = db.select(Season(season))[0] except Exception as e: log.critical( \ 'No season matching the provided season exists') sys.exit(3) else: log.critical('Must specify season with backtest') sys.exit(4) keys = {'>date' : season.getL_Bnd_Date(), \ '<date' : season.getU_Bnd_Date()} else: keys = {'>date': date, 'result': ''} try: if league: keys.update({'league': league}) order = ['<league', '<date'] fixtures = db.select(Match.createAdhoc(keys, order)) if not fixtures: raise Exception('No fixtures') except Exception as e: log.critical("Couldn't find fixtures for league and date " \ "provided, run sourcedata?") log.critical('Because...{}'.format(e)) sys.exit(5) try: if 'result' in keys: del keys['result'] if '<date' in keys: keys.update({'<match_date': keys.pop('<date')}) del keys['>date'] dt = datetime.strptime(min(f.getDate() for f in fixtures), \ '%Y-%m-%d') - timedelta(days=1) keys.update({'>match_date': dt.strftime('%Y-%m-%d')}) keys.update({'algo_id': algoId}) order = ['<league', '<match_date'] ratings = db.select(Rating.createAdhoc(keys, order)) log.debug('Num fixtures {}, ratings {}'.format(len(fixtures), \ len(ratings))) if len(fixtures) != len(ratings): raise Exception('Mismatched ratings') except Exception as e: log.critical("Couldn't find algo ratings for all fixtures, " \ "run analysematches?") log.critical('Because...{}'.format(e)) sys.exit(6) try: del keys['>match_date'] if '<match_date' in keys: del keys['<match_date'] keys.update({'>generation_date': date}) order = ['>generation_date'] stats = db.select(Statistics.createAdhoc(keys, order)) if not stats: raise Exception('No statistics') lastGenDate = stats[0].getGeneration_Date() stats = [s for s in stats if s.getGeneration_Date() == lastGenDate] except Exception as e: log.critical("Couldn't find algo statistics for league and date, " \ "run genstats?") log.critical('Because...{}'.format(e)) sys.exit(7) def statsSummary(s: Statistics): if s.getMark() == 99: return 0, 0, 0.0, 0.0, 0, 0.0, 0.0, 0, 0.0, 0.0 markF = s.getMark_Freq() homeF = s.getHome_Freq() homeP = (homeF / markF) * 100.0 if markF else 0.0 homeO = 100.0 / homeP if homeP else 99.99 drawF = s.getDraw_Freq() drawP = (drawF / markF) * 100.0 if markF else 0.0 drawO = 100.0 / drawP if drawP else 99.99 awayF = s.getAway_Freq() awayP = (awayF / markF) * 100.0 if markF else 0.0 awayO = 100.0 / awayP if awayP else 99.99 return markF, homeF, homeP, homeO, drawF, drawP, drawO, awayF, \ awayP, awayO for r in itertools.filterfalse(lambda r : r.getMark() in \ [s.getMark() for s in stats if r.getLeague() == s.getLeague()],\ ratings): stats.append(Statistics(r.getMatch_Date(), r.getAlgo_Id(), \ r.getLeague(), r.getMark(), 0, 0, 0, 0)) analytics = map(lambda r : [(r, statsSummary(s)) for s in stats \ if r.getMark() == s.getMark() \ and r.getLeague() == s.getLeague()], ratings) presentation = zip(fixtures, analytics) tables = {} mailText = 'Visit the website for more details - http://www.sweeperfootball.com<br/><br/>' for i, (league, group) in enumerate(itertools.groupby(presentation, \ lambda x : x[0].getLeague())): try: leagueDesc = db.select(League(league))[0].getDesc() except Exception as e: log.critical("Couldn't find league") log.critical('Because..{}'.format(e)) sys.exit(5) try: keys = {'league': league, 'algo_id': algoId} order = ['>config_date'] algoCfg = db.select(Algo_Config.createAdhoc(keys, order))[0] except Exception as e: log.critical("Couldn't find algo config for league") log.critical('Because...{}'.format(e)) sys.exit(6) presGrp = list(group) headers = ['Date', 'Match', 'Mark', 'M#', \ 'H#', 'H%', 'HO', 'D#', 'D%', 'DO', 'A#', 'A%', 'AO'] schema = ['{:<12}', '{:<40}', '{:>4}', '{:>4}', \ '{:>4}', '{:>5.2f}', '{:>5.2f}', '{:>4}', '{:>5.2f}', \ '{:>5.2f}', '{:>4}', '{:>5.2f}', '{:>5.2f}'] t = Table(headers=headers, schema=schema, \ title='{} Fixtures'.format(leagueDesc)) if backtest: # if we are backtesting then only return the predictions t.append([[f.getDate(), '{} (vs) {}'.format(f.getHome_Team(), \ f.getAway_Team()), r.getMark(), *a] \ for f, [(r, a)] in presGrp \ if r.getMark() > algoCfg.getL_Bnd_Mark() \ and r.getMark() < algoCfg.getU_Bnd_Mark()]) return t t.append([[f.getDate(), '{} (vs) {}'.format(f.getHome_Team(), \ f.getAway_Team()), r.getMark(), *a] \ for f, [(r, a)] in presGrp]) t.setHighlights([['Match', '{} (vs) {}'.format(f.getHome_Team(), \ f.getAway_Team()), False, False] \ for f, [(r, a)] in presGrp \ if r.getMark() > algoCfg.getL_Bnd_Mark() \ and r.getMark() < algoCfg.getU_Bnd_Mark()]) t.htmlReplacements([['(vs)', '<br/>']]) try: keys = {'>u_bnd_date': date, '<l_bnd_date': date} season = db.select(Season.createAdhoc(keys))[0].getName() except Exception as e: log.critical("Couldn't find season for date") log.critical('Because...{}'.format(e)) sys.exit(6) mask = log.getMask() log.setMask(mask & ~Logger.INFO) leagueTable, formTable = genFormTable(log, league, season, date) log.setMask(mask) log.info(t) log.info(formTable) tables[leagueDesc] = (t, leagueTable, formTable) if show: t.asHTML(show) formTable.asHTML(show) if mail: if not i: mailText += t.asHTML().replace('</body>', '') + '<br/>' else: mailText += t.asHTML(fullyFormed=False) + '<br/>' mailText += formTable.asHTML(fullyFormed=False) + '<br/>' if mail: mailText = 'MIME-Version: 1.0\nContent-type: text/html\nSubject: Sweeper Football Predictions\n\n{}</body>'.format( mailText) #mailText = 'MIME-Version: 1.0\nContent-type: text/html\nSubject: Sweeper Football Predictions - PREDICTIONS AVAILABLE FROM THIS WEEK!\n\n{}</body>'.format(mailText) mailCfg = getSweeperConfig('mail.cfg') fromAddr = mailCfg['fromAddr'] subs = db.select(Subscriber.createAdhoc({'include': 1})) toAddrs = [s.getEmail() for s in subs] server = smtplib.SMTP(mailCfg['svr'], int(mailCfg['port'])) server.ehlo() server.starttls() server.ehlo() server.login(fromAddr, mailCfg['pwd']) server.sendmail(fromAddr, toAddrs, mailText) server.quit() log.info('email sent to: {!s}'.format(toAddrs)) return tables