def get_dataframe():
session = Database.get_session()
filter = {
Util.get_field(MatchNotation.SEASON, SeasonNotation.YEAR_BEGIN): {
"$gt": 2012
},
Util.get_field(MatchNotation.FAVORITES, OddsNotation.Type.BOTH_TEAMS_TO_SCORE):
{
'$exists': 1
}
}
fields = {
MatchNotation.SEASON: 1,
MatchNotation.RESULTS: 1,
MatchNotation.ODDS_SUMMARY: 1
}
#
# matches = session.matches.find({'CHAMP.YR':{"$gt":2012},'HIT.BTS_FAV':{'$exists':1}},
# {'CHAMP':1,'FMU':1,'DT':1,'BTSR':1,'ODDS.RESUME.bts':1, 'round':1})
logger.info('Consultando... ')
matches = session.matches.list(filter, fields)
logger.info('Passando pra lista... ')
rows = list(matches)
logger.info('Normalizando... ')
df = json_normalize(rows)
logger.info('Fim')
df.to_csv('teste.csv')
logger.info('Lendo... ')
df = pd.read_csv('teste.csv')
logger.info('Lido... ')
def reset_database():
logger.info('Service: reset_database()')
session = Database.get_session()
MatchController(session).drop()
SeasonController(session).drop()
ChampionshipController(session).drop()
def _create_odds(season):
logger.info('_create_odds(): ' + str(season))
session = Database.open_new_session()
OddsController(session).create_odds(season)
OddsController(session).process_summary_odds(season)
OddsController(session).process_favorites(season)
def _create_matches(season):
session = Database.open_new_session()
logger.info('extract_matches(): ' + str(season))
MatchController(session).create_matches(season)
season = SeasonController(session).update_summary_fields(season)
MatchController(session).update_rounds(season)
def _create_matches(season):
logger.info('create_matches(): ' + str(season))
session = Database.open_new_session()
MatchController(session).create_matches(season)
SeasonController(session).update_summary_fields(season)
MatchController(session).process_results(season)
MatchController(session).update_rounds(season)
def request_matches(replace=False):
logger.info('request_matches() ')
session = Database.get_session()
seasons = SeasonController(session).list()
args = [(s, replace) for s in seasons]
pool = mp.Pool(mp.cpu_count())
pool.map(_request_matches, args)
def reset_database():
logger.info('reset_database() ')
session = Database.get_session()
ChampionshipController(session).drop()
SeasonController(session).drop()
MatchController(session).drop()
PerformanceController(session).drop()
def request_seasons(replace=False):
logger.info('request_seasons() ')
session = Database.get_session()
championships = ChampionshipController(session).list()
args = [(c, replace) for c in championships]
pool = mp.Pool(mp.cpu_count())
pool.map(_request_season, args)
def request_seasons(replace=False):
logger.info('Service: request_seasons()')
session = Database.get_session()
champs = list(ChampionshipController(session).list())
args = [(champs[idx], replace) for idx, item in enumerate(champs)]
p = mp.Pool(mp.cpu_count())
p.map(_requests_season, args)
def request_matches(replace=False):
logger.info('Service: request_matches()')
session = Database.get_session()
seasons = list(SeasonController(session).list())
args = [(seasons[i], replace) for i, item_a in enumerate(seasons)]
p = mp.Pool(mp.cpu_count())
p.map(_request_matches, args)
def create_odds(replace=False):
session = Database.get_session()
if replace:
OddsController(session).unset_odds()
seasons = SeasonController(session).list()
pool = mp.Pool(mp.cpu_count())
pool.map(_create_odds, seasons)
def test_performance(self):
session = database.get_session()
match = MatchRepository(session).get_by_id_site('bTr4iQt3')
p_filter = {
PerformanceNotation.NEXT_MATCH: match[MatchNotation.ID],
PerformanceNotation.TEAM: 'Lazio'
}
performances = PerformanceRepository(session).list(filter=p_filter)
self.assertEqual(performances.count(), 3)
for p in performances:
print(p)
if (p[PerformanceNotation.LOCAL] ==
PerformanceNotation.Local.OVERALL):
self.assertEqual(p[PerformanceNotation.POINTS], 3)
self.assertEqual(p[PerformanceNotation.WINS], 0)
self.assertEqual(p[PerformanceNotation.DRAWS], 3)
self.assertEqual(p[PerformanceNotation.LOSES], 1)
self.assertEqual(p[PerformanceNotation.MATCHES_PLAYED], 4)
self.assertEqual(p[PerformanceNotation.MATCHES_PLAYED_HOME], 2)
self.assertEqual(p[PerformanceNotation.MATCHES_PLAYED_AWAY], 2)
self.assertEqual(p[PerformanceNotation.LAST_MATCHES_NUM], 4)
self.assertEqual(p[PerformanceNotation.GOALS_FOR], 3)
self.assertEqual(p[PerformanceNotation.GOALS_AGAINST], 4)
elif p[PerformanceNotation.
LOCAL] == PerformanceNotation.Local.HOME:
self.assertEqual(p[PerformanceNotation.POINTS], 2)
self.assertEqual(p[PerformanceNotation.WINS], 0)
self.assertEqual(p[PerformanceNotation.DRAWS], 2)
self.assertEqual(p[PerformanceNotation.LOSES], 0)
self.assertEqual(p[PerformanceNotation.MATCHES_PLAYED], 4)
self.assertEqual(p[PerformanceNotation.MATCHES_PLAYED_HOME], 2)
self.assertEqual(p[PerformanceNotation.MATCHES_PLAYED_AWAY], 2)
self.assertEqual(p[PerformanceNotation.LAST_MATCHES_NUM], 2)
self.assertEqual(p[PerformanceNotation.GOALS_FOR], 2)
self.assertEqual(p[PerformanceNotation.GOALS_AGAINST], 2)
elif p[PerformanceNotation.
LOCAL] == PerformanceNotation.Local.AWAY:
self.assertEqual(p[PerformanceNotation.POINTS], 1)
self.assertEqual(p[PerformanceNotation.WINS], 0)
self.assertEqual(p[PerformanceNotation.DRAWS], 1)
self.assertEqual(p[PerformanceNotation.LOSES], 1)
self.assertEqual(p[PerformanceNotation.MATCHES_PLAYED], 4)
self.assertEqual(p[PerformanceNotation.LAST_MATCHES_NUM], 2)
self.assertEqual(p[PerformanceNotation.GOALS_FOR], 1)
self.assertEqual(p[PerformanceNotation.GOALS_AGAINST], 2)
def create_performances(replace=False):
session = Database.get_session()
if replace:
PerformanceController(session).drop()
PerformanceController(session).initialize()
seasons = SeasonController(session).list()
pool = mp.Pool(mp.cpu_count())
pool.map(_create_performances, seasons)
def create_matches(replace=False):
logger.info('create_matches() ')
session = Database.get_session()
MatchController(session).initialize()
if replace:
MatchController(session).drop()
seasons = SeasonController(session).list()
pool = mp.Pool(mp.cpu_count())
pool.map(_create_matches, seasons)
def create_markov_chains(replace=False):
session = Database.get_session()
if replace:
MarkovChainController(session).drop()
MarkovChainController(session).initialize()
seasons = SeasonController(session).list()
pool = mp.Pool(mp.cpu_count())
pool.map(_create_markov_chains, seasons)
def test_match_fields(self):
''' Checks the fields of matches '''
session = database.get_session()
match = MatchRepository(session).get_by_id_site('IXDncKQQ')
self.assertEqual(match[MatchNotation.HOME_TEAM], 'Fluminense')
self.assertEqual(match[MatchNotation.AWAY_TEAM], 'Guarani')
self.assertEqual(match[MatchNotation.HOME_GOALS], 1)
self.assertEqual(match[MatchNotation.AWAY_GOALS], 0)
self.assertEqual(
match[MatchNotation.RESULTS][OddsTypeNotation.MONEY_LINE],
OddsNotation.MoneyLine.HOME)
matchModel = Match(match)
self.assertEqual(matchModel.get_money_line_result(),
OddsNotation.MoneyLine.HOME)
self.assertEqual(matchModel.get_both_teams_to_score_result(),
OddsNotation.BothTeamsToScore.NO)
match = MatchRepository(session).get_by_id_site('l0MZkxJ0')
self.assertEqual(match[MatchNotation.HOME_TEAM], 'Goias')
self.assertEqual(match[MatchNotation.AWAY_TEAM], 'Corinthians')
self.assertEqual(match[MatchNotation.HOME_GOALS], 1)
self.assertEqual(match[MatchNotation.AWAY_GOALS], 1)
self.assertEqual(
match[MatchNotation.RESULTS][OddsTypeNotation.MONEY_LINE],
OddsNotation.MoneyLine.DRAW)
matchModel = Match(match)
self.assertEqual(matchModel.get_money_line_result(),
OddsNotation.MoneyLine.DRAW)
self.assertEqual(matchModel.get_both_teams_to_score_result(),
OddsNotation.BothTeamsToScore.YES)
match = MatchRepository(session).get_by_id_site('zJAzNd9i')
self.assertEqual(match[MatchNotation.HOME_TEAM], 'Portuguesa')
self.assertEqual(match[MatchNotation.AWAY_TEAM], 'Sao Paulo')
self.assertEqual(match[MatchNotation.HOME_GOALS], 2)
self.assertEqual(match[MatchNotation.AWAY_GOALS], 3)
self.assertEqual(
match[MatchNotation.RESULTS][OddsTypeNotation.MONEY_LINE],
OddsNotation.MoneyLine.AWAY)
matchModel = Match(match)
self.assertEqual(matchModel.get_money_line_result(),
OddsNotation.MoneyLine.AWAY)
self.assertEqual(matchModel.get_both_teams_to_score_result(),
OddsNotation.BothTeamsToScore.YES)
def test_match_odds(self):
''' Checks the odds of matches '''
session = database.get_session()
match = MatchRepository(session).get_by_id_site('8KEmQeVQ')
matchModel = Match(match)
odds_ml = matchModel.get_odds_list(OddsTypeNotation.MONEY_LINE)
odds_bts = matchModel.get_odds_list(
OddsTypeNotation.BOTH_TEAMS_TO_SCORE)
self.assertEqual(len(odds_ml), 25)
self.assertEqual(len(odds_bts), 13)
ml_summary = matchModel.get_odds_summary(OddsTypeNotation.MONEY_LINE)
bts_summary = matchModel.get_odds_summary(
OddsTypeNotation.BOTH_TEAMS_TO_SCORE)
self.assertIsNotNone(ml_summary)
self.assertIsNotNone(bts_summary)
ml_favorites = matchModel.get_odds_favorites(
OddsTypeNotation.MONEY_LINE)
bts_favorites = matchModel.get_odds_favorites(
OddsTypeNotation.BOTH_TEAMS_TO_SCORE)
self.assertIsNotNone(ml_favorites)
self.assertIsNotNone(bts_favorites)
self.assertEqual(
matchModel.get_favorite_column(OddsTypeNotation.MONEY_LINE),
OddsNotation.MoneyLine.HOME)
self.assertEqual(
matchModel.get_favorite_column(
OddsTypeNotation.BOTH_TEAMS_TO_SCORE),
OddsNotation.BothTeamsToScore.NO)
ml_hits = matchModel.get_hits(OddsTypeNotation.MONEY_LINE)
bts_hits = matchModel.get_hits(OddsTypeNotation.BOTH_TEAMS_TO_SCORE)
self.assertEqual(ml_hits[OddsFavoritesNotation.FAVORITE], True)
self.assertEqual(bts_hits[OddsFavoritesNotation.FAVORITE], False)
def test_first_match(self):
session = database.get_session()
match = MatchRepository(session).get_by_id_site('OvMLBTBD')
p_filter = {PerformanceNotation.NEXT_MATCH: match[MatchNotation.ID]}
markov_chains = MarkovChainRepository(session).list(filter=p_filter)
self.assertEqual(markov_chains.count(), 6)
for mc in markov_chains:
self.assertEqual(mc[PerformanceNotation.MATCHES_PLAYED], 0)
self.assertEqual(mc[PerformanceNotation.MATCHES_PLAYED_HOME], 0)
self.assertEqual(mc[PerformanceNotation.MATCHES_PLAYED_AWAY], 0)
self.assertEqual(mc[PerformanceNotation.LAST_MATCHES_NUM], 0)
self.assertIn(mc[PerformanceNotation.TEAM],['Lazio','Torino'])
chains = mc[MarkovChainNotation.CHAINS]
self.assertEqual(len(chains),2)
def test_first_match(self):
session = database.get_session()
match = MatchRepository(session).get_by_id_site('OvMLBTBD')
p_filter = {PerformanceNotation.NEXT_MATCH: match[MatchNotation.ID]}
performances = PerformanceRepository(session).list(filter=p_filter)
self.assertEqual(performances.count(), 6)
for p in performances:
self.assertEqual(p[PerformanceNotation.POINTS], 0)
self.assertEqual(p[PerformanceNotation.WINS], 0)
self.assertEqual(p[PerformanceNotation.DRAWS], 0)
self.assertEqual(p[PerformanceNotation.LOSES], 0)
self.assertEqual(p[PerformanceNotation.MATCHES_PLAYED], 0)
self.assertEqual(p[PerformanceNotation.MATCHES_PLAYED_HOME], 0)
self.assertEqual(p[PerformanceNotation.MATCHES_PLAYED_AWAY], 0)
self.assertEqual(p[PerformanceNotation.LAST_MATCHES_NUM], 0)
self.assertEqual(p[PerformanceNotation.GOALS_FOR], 0)
self.assertEqual(p[PerformanceNotation.GOALS_AGAINST], 0)
self.assertIn(p[PerformanceNotation.TEAM], ['Lazio', 'Torino'])
def _request_matches(args):
logger.info('create_matches(): ' + str(args))
session = Database.open_new_session()
MatchController(session).crawler(*args)
def _process_hits(season):
logger.info('_process_hits(): ' + str(season))
session = Database.open_new_session()
MatchController(session).process_hits(season)
def create_seasons(initial_year, final_year):
logger.info('Service: create_seasons()')
session = Database.get_session()
SeasonController(session).initialize(initial_year, final_year)
def create_championships():
logger.info('Service: create_championships()')
session = Database.get_session()
ChampionshipController(session).initialize()
def _request_season(args):
logger.info('_request_season() ' + str(args))
session = Database.open_new_session()
SeasonController(session).crawler(*args)
def _request_matches(args):
session = Database.open_new_session()
SeasonController(session).crawler_matches(*args)
def create_dataframe(replace=False):
session = Database.get_session()
DataframeController(session).create_dataframe(replace)
def create_seasons(initial_year, final_year):
""" Create seasons """
logger.info('create_seasons() ')
session = Database.get_session()
SeasonController(session).initialize(initial_year, final_year)
def create_championships():
""" Create championships """
logger.info('create_championships() ')
session = Database.get_session()
ChampionshipController(session).initialize()
def _create_markov_chains(season):
logger.info('_create_markov_chains: ' + str(season))
session = Database.open_new_session()
MarkovChainController(session).create_markov_chains(season)
def _create_performances(season):
logger.info('_create_performances: ' + str(season))
session = Database.open_new_session()
PerformanceController(session).create_performances(season)