def run_experiment_0(exp, league, type_evaluation, **params):
"""
:param exp:
:param league:
:param type_evaluation:
:param params:
:return:
"""
predictor = Predictor.get_predictor()
filter_season = util.get_default(params, "season", None)
for season in league.get_seasons():
if not util.is_None(filter_season) and season != filter_season:
continue
invest = 0
profit = 0
print(season)
if season == util.get_current_season():
break
for stage in range(1, league.get_stages_by_season(season) + 1):
# KEY: match id VALUE: <prediction, probability>
stage_predictions = predictor.predict(league, season, stage,
**params)
current_stage_bet = StageBet(stage, type_evaluation)
for match_id, pair in stage_predictions.items():
if len(pair) == 0:
continue
match = Match.read_by_match_id(match_id)
if util.is_None(match.B365H) or util.is_None(
match.B365D) or util.is_None(match.B365A):
continue
predicted_label, prob = pair[0], pair[1]
bet_odd = get_bet_odd(predicted_label, match)
m_invest, m_profit = evaluate_bet(predictor, type_evaluation,
match, predicted_label, prob)
if type_evaluation == 5:
if m_invest == 1:
current_stage_bet.add_bet(prob, m_profit, bet_odd)
elif type_evaluation == 6:
current_stage_bet.add_bet(prob, m_profit, bet_odd)
elif m_invest == 1:
current_stage_bet.add_bet(prob, m_profit)
profit += current_stage_bet.get_profit()
invest += current_stage_bet.get_invest()
print(stage, "\t",
str(round(profit - invest, 2)).replace(".", ","))
print("Final investment:\t", str(round(invest, 2)).replace(".", ","))
print("Final profit:\t", str(round(profit, 2)).replace(".", ","))
def write_new_player(player_name,
player_fifa_api_id,
birthday,
height,
weight,
player_api_id=None):
"""
Insert a new player in the DB
:param player_name:
:param player_fifa_api_id:
:param birthday:
:param height:
:param weight:
:param player_api_id:
:return:
"""
print("Inserting new player", player_name, player_api_id,
player_fifa_api_id)
player_diz = dict()
player_diz["player_name"] = player_name
if not util.is_None(player_fifa_api_id):
player_diz["player_fifa_api_id"] = player_fifa_api_id
if not util.is_None(birthday):
player_diz["birthday"] = birthday
if not util.is_None(height):
player_diz["height"] = height
if not util.is_None(weight):
player_diz["weight"] = weight
if not util.is_None(player_api_id):
player_diz["player_api_id"] = player_api_id
SQLLite.get_connection().insert("Player", player_diz)
return read_by_fifa_api_id(player_fifa_api_id)
def check_player(self, player_name, player_fifa_api_id, birthday, height,
weight):
"""
This method can be called if and only if self.player is set
It aims to check every attribute of the player that may be not set!
:param player_name:
:param player_fifa_api_id:
:param birthday:
:param height:
:param weight:
:return:
"""
# check player_name
if util.is_None(self.player.player_name):
self.player.player_name = player_name
elif player_name not in self.player.player_name:
self.player.player_name = self.player.player_name + "|" + player_name
# check player_fifa_api_id
if util.is_None(self.player.player_fifa_api_id):
self.player.player_fifa_api_id = player_fifa_api_id
# check birthday
if util.is_None(self.player.birthday):
self.player.birthday = birthday
# check height
if util.is_None(self.player.height):
self.player.height = height
# check weight
if util.is_None(self.player.weight):
self.player.weight = weight
def start_crawling(self):
"""
Start crawling this team
:return:
"""
if util.is_None(
self.team) or (not util.is_None(self.team)
and util.is_None(self.team.team_fifa_api_id)):
# If one of the follow:
# 1) team not stored in the DB
# 2) Team fifa api id not stored in the DB
# --> looking for name and fifa_api_id
team_long_name, team_fifa_api_id = self.look_for_base_data()
if util.is_None(self.team):
# team not present in the DB
self.team = Team.write_new_team(team_long_name,
team_fifa_api_id)
else:
# team present in the DB, but without set the team_fifa_api_id
self.team.team_fifa_api_id = team_fifa_api_id
self.team = Team.update(self.team)
# looking for players belonging this team
link_players_found = self.look_for_players()
for player_link, player_name in link_players_found.items():
player_fifa_api_id = player_link[25:]
player = Player.read_by_fifa_api_id(player_fifa_api_id)
# crawl the player if and only if on of the following happens:
# 1) PLAYER DOES NOT EXIST IN THE DB
# 2) PLAYER ATTRIBUTES DO NOT EXIST IN THE DB
# 3) PLAYER ATTRIBUTES IN THE DB ARE OLD
# 4) FORCE PARSING OF PLAYER ATTRIBUTES
if \
not player \
or not player.get_last_player_attributes() \
or util.compare_time_to_now(player.get_last_player_attributes().date, self.day_passed) \
or self.force_parsing:
log.debug("Player to crawl [" + player_link + ", " +
player_name + "]")
cp = CrawlerPlayer(player, player_link)
cp.start_crawling()
# looking for build up play
attributes_found = self.look_for_team_attributes()
if len(attributes_found) > 0:
self.team.save_team_attributes(attributes_found)
def select_like(self,
table_name,
column_filter='*',
columns_order=None,
**id):
id_condition = ""
if len(id) > 0:
id_condition = "WHERE "
for attrbiute, value in id.items():
id_condition += attrbiute + " like '%" + str(value).replace(
"'", "''") + "%' AND "
id_condition = id_condition[0:-4]
if column_filter == '*':
column_names = self.getColumnFromTable(table_name)
else:
column_names = column_filter.split(",")
if util.is_None(columns_order):
order_by = ""
else:
order_by = " ORDER BY " + columns_order
row_results = []
select_like = "SELECT " + column_filter + " FROM " + table_name + " " + id_condition + order_by + ";"
log.debug("Select like [" + select_like + "]")
for sqllite_row in self.cursor.execute(select_like):
if sqllite_row is None:
continue
row = {}
for i, name in enumerate(column_names):
row[name] = sqllite_row[i]
row_results.append(row)
return row_results
def get_match_event_out(match):
predictor = Predictor.get_current_predictor()
match_event_str = ""
match_event_str += match.get_home_team(
).team_long_name + " vs " + match.get_away_team().team_long_name
match_event_str += "\n" + "[" + match.date + "]"
match_event = match.get_match_event()
if util.is_None(match_event):
match_event_str += "\nNo bet-odds found"
return match_event_str
bet_events_dict = match_event.get_last_bet_values()
for bet_event_name, bet_event in bet_events_dict.items():
prediction = None
if bet_event_name == "Match Result":
try:
prediction = predictor.predictions[match.get_league().id][
match.id][0]
except KeyError:
pass
match_event_str += get_bet_event_out(bet_event, prediction)
return match_event_str
def get_ranking(self, season, home=None):
"""
Return the position of the teams in this league
Return a list of pair <Points, Team>, ordered py descending points
:param season:
:param stage:
:param home:
:return:
"""
import src.application.Domain.Team as Team
matches = self.get_matches(season=season, finished=True)
teams = self.get_teams(season=season)
ranking = {team.id: 0 for team in teams}
for m in matches:
result, winner = m.get_winner()
if not util.is_None(winner):
ranking[winner.id] += 3
else:
ranking[m.get_home_team().id] += 1
ranking[m.get_away_team().id] += 1
ranking_ret = []
for team, p in sorted(ranking.items(),
key=operator.itemgetter(1))[::-1]:
ranking_ret.append((p, Team.read_by_id(team)))
return ranking_ret
def read_by_api_id(player_api_id):
"""
Read a player by its api_id
:param player_api_id:
:return:
"""
if util.is_None(player_api_id):
return None
try:
return Cache.get_element(player_api_id, "PLAYER_BY_API_ID")
except KeyError:
pass
filter = {"player_api_id": player_api_id}
try:
sqllite_row = SQLLite.get_connection().select("Player", **filter)[0]
except IndexError:
return None
player = Player(sqllite_row["id"])
for attribute, value in sqllite_row.items():
player.__setattr__(attribute, value)
Cache.add_element(player.player_fifa_api_id, player,
"PLAYER_BY_FIFA_API_ID")
Cache.add_element(player.player_api_id, player, "PLAYER_BY_API_ID")
Cache.add_element(player.player_name, player, "PLAYER_BY_NAME")
Cache.add_element(player.id, player, "PLAYER_BY_ID")
return player
def get_printable_prediction(match, prediction, probability):
out_prediction = ""
out_prediction += match.get_home_team().team_long_name+" vs " + match.get_away_team().team_long_name
out_prediction += "\n"+str(prediction)+"\t("+str(round(probability*100, 2))+"%):"
match_event = match.get_match_event()
if util.is_None(match_event):
return out_prediction
bet_event = match_event.get_last_bet_values(event_name="Match Result")["Match Result"]
if not util.is_None(bet_event):
bookmaker_bet_odd = bet_event.get_bet_odds_by_bet(prediction)
if not util.is_None(bookmaker_bet_odd):
out_prediction += "\t"+str(bookmaker_bet_odd)+"\t("+str(round(100/bookmaker_bet_odd, 2))+"%)"
out_prediction += "\nBet > "+str(probability > 1/bookmaker_bet_odd)
out_prediction += ""
return out_prediction
def get_assist_done(self, season=None, stage=None):
"""
Return the number of assist this player has done
:param season:
:param stage:
:return:
"""
cnt = 0
for m in self.get_matches(season=season, ordered=True):
if util.is_None(m.goal):
continue
if not util.is_None(stage) and m.stage >= stage:
return cnt
soup = BeautifulSoup(m.goal, "html.parser")
for player1 in soup.find_all('player2'):
if int(str(player1.string).strip()) == self.player_api_id:
cnt += 1
return cnt
def run_experiment_4(exp,
league,
predictor=Predictor.get_predictor(),
**params):
for season in league.get_seasons()[4:]:
distance = {i: 0 for i in [0, 1, 2]}
number_bet_odds = {i: 0 for i in [0, 1, 2]}
print(season)
for stage in range(1, league.get_stages_by_season(season) + 1):
stage_predictions = predictor.predict(league, season, stage,
**params)
for match_id, pair in stage_predictions.items():
if len(pair) == 0:
continue
match = Match.read_by_match_id(match_id)
if util.is_None(match.B365H) or util.is_None(match.B365D) or util.is_None(match.B365A)\
or match.B365H == 0 or match.B365D == 0 or match.B365A == 0:
continue
predicted_label, prob = pair[0], pair[1]
our_bet_odds = 1 / prob
bm_bet_odds = -1
if predicted_label == 1:
bm_bet_odds = match.B365H
elif predicted_label == 0:
bm_bet_odds = match.B365D
elif predicted_label == 2:
bm_bet_odds = match.B365A
distance[predicted_label] += math.fabs(our_bet_odds -
bm_bet_odds)
number_bet_odds[predicted_label] += 1
print(1, distance[1] / max(number_bet_odds[1], 1))
print(0, distance[0] / max(number_bet_odds[0], 1))
print(2, distance[2] / max(number_bet_odds[2], 1))
def look_for_matches(self, date, force_parsing=False):
print("Elaborating matches of the date:", date)
matches_link = self.host_url_match + "/sport_events/1%2F"+date+"%2Fbasic_h2h%2F0%2F0/"
log.debug("Looking for matches of date ["+date+"] at link ["+matches_link+"]")
try:
page = requests.get(matches_link).text
except Exception as e:
print(e)
soup = BeautifulSoup(page, "html.parser")
header_list = soup.find_all('div', {'class': 'mx-default-header mx-text-align-left mx-flexbox-container '})
body_list = soup.find_all('div',
{'class': 'mx-table mx-soccer mx-matches-table mx-group-by-stage mx-container mx-league mx-livescore-table'})
for header, body in zip(header_list, body_list):
# reading the league
# Notice that the league is identified also with an attribute called "data-stage"
league_name = str(header.a.string).strip()
league_data_stage = header.a.attrs['data-stage']
# check if the this league corresponds to one of those one managed!
cl = CrawlerLeague(league_name, league_data_stage)
if cl.is_in_a_managed_country() and len(league_name) > 3:
league = cl.get_league()
if util.is_None(league):
log.warning("Impossible to crawl this league [" + league_name + ", " + league_data_stage + "]")
continue
print("\t- Looking for the league [" + league.name + "]")
season = cl.get_season()
for div_event in body.find_all('div', {'class': 'mx-stage-events'}):
# event correspond to "match_api_id"
event = str(div_event.attrs["class"][3]).split("-")[2]
match = Match.read_by_match_api_id(event)
if force_parsing \
or not match \
or not match.are_teams_linedup() \
or not match.are_incidents_managed() \
or not match.get_home_team() \
or not match.get_away_team():
# crawl when at least one of the following happen:
# - match is not in the DB
# - formation of the teams are not in the DB
# - incidents of the match are not in the DB
# - home_team_api_id is not matched to any team in the DB
# - away_team_api_id is not matched to any team in the DB
log.debug("Need to crawl match ["+event+"]")
cm = CrawlerMatch(match, league, event)
cm.parse_json(season)
else:
log.debug("Not need to crawl match [" + event + "]")
def read_players_api_id_by_team_api_id(team_api_id, season=None):
"""
return a list of player_api_id
if season is set, consider only that list
:param team_api_id:
:param season:
:return:
"""
players_api_id = set()
filter = {}
if season:
filter["season"] = season
else:
season = ""
try:
return Cache.get_element(
str(team_api_id) + "_" + season,
"MATCH_GET_PLAYERS_BY_TEAM_API_ID")
except KeyError:
pass
filter["home_team_api_id"] = team_api_id
for sqllite_row in SQLLite.get_connection().select(
"Match",
column_filter="home_player_1, home_player_2, "
"home_player_3, home_player_4, "
"home_player_5, home_player_6, "
"home_player_7, home_player_8, "
"home_player_9, home_player_10, "
"home_player_11",
**filter):
for home_player_i, player_api_id in sqllite_row.items():
if player_api_id:
players_api_id.add(player_api_id)
del (filter["home_team_api_id"])
filter["away_team_api_id"] = team_api_id
for sqllite_row in SQLLite.get_connection().select(
"Match",
column_filter="away_player_1, away_player_2, "
"away_player_3, away_player_4, "
"away_player_5, away_player_6, "
"away_player_7, away_player_8, "
"away_player_9, away_player_10, "
"away_player_11",
**filter):
for away_player_i, player_api_id in sqllite_row.items():
if not util.is_None(player_api_id):
players_api_id.add(player_api_id)
Cache.add_element(
str(team_api_id) + "_" + season, players_api_id,
"MATCH_GET_PLAYERS_BY_TEAM_API_ID")
return players_api_id
def are_incidents_managed(self):
"""
Return TURE if and only if all the incidents are managed
:return:
"""
if util.is_None(self.goal) \
or util.is_None(self.shoton) \
or util.is_None(self.shotoff) \
or util.is_None(self.foulcommit) \
or util.is_None(self.card) \
or util.is_None(self.cross) \
or util.is_None(self.corner) \
or util.is_None(self.possession):
return False
return True
def update(self, table_name, object):
column_names = self.getColumnFromTable(table_name)
update = "UPDATE " + table_name + " SET "
for column in column_names:
value = str(object.__getattribute__(column)).replace("'", "''")
if util.is_None(value):
continue
update += column + "='" + value + "',"
update = update[:-1]
update += " WHERE id = '" + str(object.id) + "'"
self.execute_update(update)
def read_by_team_api_id(team_api_id, season=None):
"""
Return list of players that play in the team identified my team_api_id
if season is set, consider only that season
:param team_api_id:
:param season:
:return:
"""
if not season:
season = ""
try:
return Cache.get_element(
str(team_api_id) + "_" + season, "PLAYER_BY_TEAM_API_ID")
except KeyError:
pass
players = []
players_api_id = Match.read_players_api_id_by_team_api_id(
team_api_id, season)
for player_api_id in players_api_id:
# if the player_api_id is not set --> continue
if util.is_None(player_api_id):
continue
try:
player = Cache.get_element(player_api_id, "PLAYER_BY_API_ID")
except KeyError:
filter = {"player_api_id": player_api_id}
try:
sqllite_row = SQLLite.get_connection().select(
"Player", **filter)[0]
except IndexError:
log.warning("Player api id not found in DB [" +
str(player_api_id) + "]")
continue
player = Player(sqllite_row["id"])
for attribute, value in sqllite_row.items():
player.__setattr__(attribute, value)
Cache.add_element(player_api_id, player, "PLAYER_BY_API_ID")
players.append(player)
Cache.add_element(
str(team_api_id) + "_" + season, players, "PLAYER_BY_TEAM_API_ID")
return players
def get_matches(self, season=None, ordered=True, stage=None):
"""
Return the matches this player has played
:param season:
:param ordered:
:param stage:
:return:
"""
if util.is_None(self.player_api_id):
return []
matches = Match.read_by_player_api_id(self.player_api_id)
if season:
matches = [m for m in matches if m.season == season]
if stage:
matches = [m for m in matches if m.stage < stage]
if ordered:
matches = sorted(matches, key=lambda match: match.stage)
return matches
def get_goal_received(self, season=None, stage=None):
"""
Return the goal received by this player
:param season:
:param stage:
:return:
"""
cnt = 0
current_team = self.get_current_team()
matches = self.get_matches(season=season, ordered=True)
for m in matches:
if not util.is_None(stage) and m.stage >= stage:
return cnt
if m.home_team_api_id == current_team.team_api_id:
cnt += m.away_team_goal
else:
cnt += m.home_team_goal
return cnt
def get_formation(match, team, home=True):
formation_str = "\n\n" + team.team_short_name
# get trends
formation_str += get_team_trend_str(match, team)
# get Goals
formation_str += get_team_goals_str(match, team)
formation_str += get_team_goals_str(match, team, n=5)
if home:
team_lines = match.get_home_team_lines_up()
else:
team_lines = match.get_away_team_lines_up()
team_goal_done, team_goal_received, num_matches = team.get_goals_by_season_and_stage(
util.get_current_season(), match.stage)
for i, player in enumerate(team_lines):
if not util.is_None(player):
if player.is_gk():
goal_dr = player.get_goal_received(
season=util.get_current_season(), stage=match.stage)
team_goal_dr = team_goal_received
else:
goal_dr = player.get_goal_done(
season=util.get_current_season(), stage=match.stage)
team_goal_dr = team_goal_done
formation_str += '\n\t' + str(i + 1) + ") " + player.player_name.ljust(25, ".") \
+ "\t P: "+str(len(player.get_matches(season=util.get_current_season(), stage=match.stage))) + '/' + str(match.stage-1) \
+ "\t G: "+str(goal_dr)+"/"+str(team_goal_dr)
if not player.is_gk():
assist_done = player.get_assist_done(
season=util.get_current_season(), stage=match.stage)
team_assist_done = team.get_assist_by_season_and_stage(
season=util.get_current_season(), stage=match.stage)
formation_str += "\t A: " + str(assist_done) + "/" + str(
team_assist_done)
return formation_str
def is_finished(self):
"""
finished if and only if incidents of the match are managed
:return:
"""
is_scored_goals = self.home_team_goal > 0 or self.away_team_goal > 0
if (is_scored_goals and util.compare_time_to_now(self.date, 1)) \
or (not is_scored_goals and util.compare_time_to_now(self.date, 100)):
return True
if util.is_None(self.goal) \
and util.is_None(self.shoton) \
and util.is_None(self.shotoff) \
and util.is_None(self.foulcommit) \
and util.is_None(self.card) \
and util.is_None(self.cross) \
and util.is_None(self.corner) \
and util.is_None(self.possession):
return False
return True
def read_by_match_event_id_and_event_name(match_event_id, event_name=None):
'''
Return the list of bet_event of the input match
:param match_event_id:
:param event_name:
:return:
'''
filter = {}
filter["match_event_id"] = str(match_event_id)
if not util.is_None(event_name):
filter["event_name"] = event_name
bet_events = []
for sqllite_row in SQLLite.get_connection().select("Bet_Event", **filter):
be = BetEvent(sqllite_row["id"])
for attribute, value in sqllite_row.items():
be.__setattr__(attribute, value)
bet_events.append(be)
return bet_events
def get_matches(self,
season=None,
ordered=True,
date=None,
finished=None,
stage=None):
"""
return the matches belonging to this league
:param season:
:param ordered:
:param date:
:param finished:
:param stage:
:return:
"""
matches = Match.read_matches_by_league(self.id, season)
if ordered:
try:
matches = sorted(matches, key=lambda match: match.stage)
except TypeError as e:
log.error("Impossible to order match of the league [" +
str(self.id) + "], of the season [" + season +
"] by stage")
raise e
if not util.is_None(finished) and finished:
matches = [m for m in matches if m.is_finished()]
if stage:
matches = [m for m in matches if m.stage == stage]
if date:
matches = [m for m in matches if m.date.startswith(date)]
return matches
def get_training_ranking(self,
season,
stage_to_predict,
stages_to_train,
home=None):
"""
:param season:
:param stage_to_predict:
:param stages_to_train:
:param home:
:return:
"""
import src.application.Domain.Team as Team
matches = self.get_training_matches(season, stage_to_predict,
stages_to_train)
teams = self.get_teams(season=season)
ranking = {team.id: 0 for team in teams}
num_matches = {team.id: 0 for team in teams}
for m in matches:
try:
result, winner = m.get_winner()
if result == 0:
winner_id = None
else:
winner_id = winner.id
home_id = m.get_home_team().id
away_id = m.get_away_team().id
except AttributeError:
continue
if util.is_None(home):
# total ranking
util.increase_dict_entry(home_id, num_matches)
util.increase_dict_entry(away_id, num_matches)
if result != 0:
util.increase_dict_entry(winner_id, ranking, 3)
else:
util.increase_dict_entry(home_id, ranking, 1)
util.increase_dict_entry(away_id, ranking, 1)
elif home:
# home ranking
util.increase_dict_entry(home_id, num_matches)
if result == 1:
util.increase_dict_entry(winner_id, ranking, 3)
elif result == 0:
util.increase_dict_entry(home_id, ranking, 1)
else:
# away ranking
util.increase_dict_entry(away_id, num_matches)
if result == 2:
util.increase_dict_entry(winner_id, ranking, 3)
elif result == 0:
util.increase_dict_entry(away_id, ranking, 1)
# divide the overall point by the number of matches done
norm_ranking = dict()
for team_id, points in ranking.items():
if num_matches[team_id] > 0:
norm_ranking[team_id] = points / num_matches[team_id]
else:
norm_ranking[team_id] = 0
# order the normalized rank
ranking_ret = []
for team, p in sorted(norm_ranking.items(),
key=operator.itemgetter(1))[::-1]:
ranking_ret.append((p, Team.read_by_id(team)))
return ranking_ret
def get_training_matches(self,
season,
stage_to_predict,
stages_to_train,
consider_last=False):
"""
Return the training matches needed by the machine learning algorithm
:param season:
:param stage_to_predict:
:param stages_to_train:
:param consider_last:
:return:
"""
if util.is_None(stages_to_train):
# stages to train not defined --> return only stage of this season
return [
m for m in self.get_matches(
season=season, ordered=True, finished=True)
if m.stage < stage_to_predict
]
else:
# stages to train is defined --> return all the matches played it those number of stages
# EX: stage_to_predict = 7, n_match_1_stage = 10 --> return 70 matches
if consider_last:
# start to consider matches of previous seasons --> take them in the reverse order
training_matches = [
m for m in self.get_matches(
season=season, ordered=True, finished=True)
]
training_matches = training_matches[::-1]
else:
training_matches = [
m for m in self.get_matches(
season=season, ordered=True, finished=True)
if m.stage < stage_to_predict
]
if len(training_matches) == 0 and season < '2006/2007':
raise MLException(0)
stages_training = set([(m.stage, m.season)
for m in training_matches])
if len(stages_training) < stages_to_train:
# need more matches from the past season, considering the last matches
past_training_matches = self.get_training_matches(
util.get_previous_season(season),
0,
stages_to_train - len(stages_training),
consider_last=True)
training_matches.extend(past_training_matches)
n_matches_in_stage = int(len(self.get_teams_current_season()) / 2)
if len(training_matches) / n_matches_in_stage > stages_to_train:
# too matches in training --> remove too far
if consider_last:
return training_matches[:stages_to_train *
n_matches_in_stage][::-1]
else:
return training_matches[-stages_to_train *
n_matches_in_stage:]
return training_matches
