def run_experiment_0(exp, league, type_evaluation, **params): """ :param exp: :param league: :param type_evaluation: :param params: :return: """ predictor = Predictor.get_predictor() filter_season = util.get_default(params, "season", None) for season in league.get_seasons(): if not util.is_None(filter_season) and season != filter_season: continue invest = 0 profit = 0 print(season) if season == util.get_current_season(): break for stage in range(1, league.get_stages_by_season(season) + 1): # KEY: match id VALUE: <prediction, probability> stage_predictions = predictor.predict(league, season, stage, **params) current_stage_bet = StageBet(stage, type_evaluation) for match_id, pair in stage_predictions.items(): if len(pair) == 0: continue match = Match.read_by_match_id(match_id) if util.is_None(match.B365H) or util.is_None( match.B365D) or util.is_None(match.B365A): continue predicted_label, prob = pair[0], pair[1] bet_odd = get_bet_odd(predicted_label, match) m_invest, m_profit = evaluate_bet(predictor, type_evaluation, match, predicted_label, prob) if type_evaluation == 5: if m_invest == 1: current_stage_bet.add_bet(prob, m_profit, bet_odd) elif type_evaluation == 6: current_stage_bet.add_bet(prob, m_profit, bet_odd) elif m_invest == 1: current_stage_bet.add_bet(prob, m_profit) profit += current_stage_bet.get_profit() invest += current_stage_bet.get_invest() print(stage, "\t", str(round(profit - invest, 2)).replace(".", ",")) print("Final investment:\t", str(round(invest, 2)).replace(".", ",")) print("Final profit:\t", str(round(profit, 2)).replace(".", ","))
def write_new_player(player_name, player_fifa_api_id, birthday, height, weight, player_api_id=None): """ Insert a new player in the DB :param player_name: :param player_fifa_api_id: :param birthday: :param height: :param weight: :param player_api_id: :return: """ print("Inserting new player", player_name, player_api_id, player_fifa_api_id) player_diz = dict() player_diz["player_name"] = player_name if not util.is_None(player_fifa_api_id): player_diz["player_fifa_api_id"] = player_fifa_api_id if not util.is_None(birthday): player_diz["birthday"] = birthday if not util.is_None(height): player_diz["height"] = height if not util.is_None(weight): player_diz["weight"] = weight if not util.is_None(player_api_id): player_diz["player_api_id"] = player_api_id SQLLite.get_connection().insert("Player", player_diz) return read_by_fifa_api_id(player_fifa_api_id)
def check_player(self, player_name, player_fifa_api_id, birthday, height, weight): """ This method can be called if and only if self.player is set It aims to check every attribute of the player that may be not set! :param player_name: :param player_fifa_api_id: :param birthday: :param height: :param weight: :return: """ # check player_name if util.is_None(self.player.player_name): self.player.player_name = player_name elif player_name not in self.player.player_name: self.player.player_name = self.player.player_name + "|" + player_name # check player_fifa_api_id if util.is_None(self.player.player_fifa_api_id): self.player.player_fifa_api_id = player_fifa_api_id # check birthday if util.is_None(self.player.birthday): self.player.birthday = birthday # check height if util.is_None(self.player.height): self.player.height = height # check weight if util.is_None(self.player.weight): self.player.weight = weight
def start_crawling(self): """ Start crawling this team :return: """ if util.is_None( self.team) or (not util.is_None(self.team) and util.is_None(self.team.team_fifa_api_id)): # If one of the follow: # 1) team not stored in the DB # 2) Team fifa api id not stored in the DB # --> looking for name and fifa_api_id team_long_name, team_fifa_api_id = self.look_for_base_data() if util.is_None(self.team): # team not present in the DB self.team = Team.write_new_team(team_long_name, team_fifa_api_id) else: # team present in the DB, but without set the team_fifa_api_id self.team.team_fifa_api_id = team_fifa_api_id self.team = Team.update(self.team) # looking for players belonging this team link_players_found = self.look_for_players() for player_link, player_name in link_players_found.items(): player_fifa_api_id = player_link[25:] player = Player.read_by_fifa_api_id(player_fifa_api_id) # crawl the player if and only if on of the following happens: # 1) PLAYER DOES NOT EXIST IN THE DB # 2) PLAYER ATTRIBUTES DO NOT EXIST IN THE DB # 3) PLAYER ATTRIBUTES IN THE DB ARE OLD # 4) FORCE PARSING OF PLAYER ATTRIBUTES if \ not player \ or not player.get_last_player_attributes() \ or util.compare_time_to_now(player.get_last_player_attributes().date, self.day_passed) \ or self.force_parsing: log.debug("Player to crawl [" + player_link + ", " + player_name + "]") cp = CrawlerPlayer(player, player_link) cp.start_crawling() # looking for build up play attributes_found = self.look_for_team_attributes() if len(attributes_found) > 0: self.team.save_team_attributes(attributes_found)
def select_like(self, table_name, column_filter='*', columns_order=None, **id): id_condition = "" if len(id) > 0: id_condition = "WHERE " for attrbiute, value in id.items(): id_condition += attrbiute + " like '%" + str(value).replace( "'", "''") + "%' AND " id_condition = id_condition[0:-4] if column_filter == '*': column_names = self.getColumnFromTable(table_name) else: column_names = column_filter.split(",") if util.is_None(columns_order): order_by = "" else: order_by = " ORDER BY " + columns_order row_results = [] select_like = "SELECT " + column_filter + " FROM " + table_name + " " + id_condition + order_by + ";" log.debug("Select like [" + select_like + "]") for sqllite_row in self.cursor.execute(select_like): if sqllite_row is None: continue row = {} for i, name in enumerate(column_names): row[name] = sqllite_row[i] row_results.append(row) return row_results
def get_match_event_out(match): predictor = Predictor.get_current_predictor() match_event_str = "" match_event_str += match.get_home_team( ).team_long_name + " vs " + match.get_away_team().team_long_name match_event_str += "\n" + "[" + match.date + "]" match_event = match.get_match_event() if util.is_None(match_event): match_event_str += "\nNo bet-odds found" return match_event_str bet_events_dict = match_event.get_last_bet_values() for bet_event_name, bet_event in bet_events_dict.items(): prediction = None if bet_event_name == "Match Result": try: prediction = predictor.predictions[match.get_league().id][ match.id][0] except KeyError: pass match_event_str += get_bet_event_out(bet_event, prediction) return match_event_str
def get_ranking(self, season, home=None): """ Return the position of the teams in this league Return a list of pair <Points, Team>, ordered py descending points :param season: :param stage: :param home: :return: """ import src.application.Domain.Team as Team matches = self.get_matches(season=season, finished=True) teams = self.get_teams(season=season) ranking = {team.id: 0 for team in teams} for m in matches: result, winner = m.get_winner() if not util.is_None(winner): ranking[winner.id] += 3 else: ranking[m.get_home_team().id] += 1 ranking[m.get_away_team().id] += 1 ranking_ret = [] for team, p in sorted(ranking.items(), key=operator.itemgetter(1))[::-1]: ranking_ret.append((p, Team.read_by_id(team))) return ranking_ret
def read_by_api_id(player_api_id): """ Read a player by its api_id :param player_api_id: :return: """ if util.is_None(player_api_id): return None try: return Cache.get_element(player_api_id, "PLAYER_BY_API_ID") except KeyError: pass filter = {"player_api_id": player_api_id} try: sqllite_row = SQLLite.get_connection().select("Player", **filter)[0] except IndexError: return None player = Player(sqllite_row["id"]) for attribute, value in sqllite_row.items(): player.__setattr__(attribute, value) Cache.add_element(player.player_fifa_api_id, player, "PLAYER_BY_FIFA_API_ID") Cache.add_element(player.player_api_id, player, "PLAYER_BY_API_ID") Cache.add_element(player.player_name, player, "PLAYER_BY_NAME") Cache.add_element(player.id, player, "PLAYER_BY_ID") return player
def get_printable_prediction(match, prediction, probability): out_prediction = "" out_prediction += match.get_home_team().team_long_name+" vs " + match.get_away_team().team_long_name out_prediction += "\n"+str(prediction)+"\t("+str(round(probability*100, 2))+"%):" match_event = match.get_match_event() if util.is_None(match_event): return out_prediction bet_event = match_event.get_last_bet_values(event_name="Match Result")["Match Result"] if not util.is_None(bet_event): bookmaker_bet_odd = bet_event.get_bet_odds_by_bet(prediction) if not util.is_None(bookmaker_bet_odd): out_prediction += "\t"+str(bookmaker_bet_odd)+"\t("+str(round(100/bookmaker_bet_odd, 2))+"%)" out_prediction += "\nBet > "+str(probability > 1/bookmaker_bet_odd) out_prediction += "" return out_prediction
def get_assist_done(self, season=None, stage=None): """ Return the number of assist this player has done :param season: :param stage: :return: """ cnt = 0 for m in self.get_matches(season=season, ordered=True): if util.is_None(m.goal): continue if not util.is_None(stage) and m.stage >= stage: return cnt soup = BeautifulSoup(m.goal, "html.parser") for player1 in soup.find_all('player2'): if int(str(player1.string).strip()) == self.player_api_id: cnt += 1 return cnt
def run_experiment_4(exp, league, predictor=Predictor.get_predictor(), **params): for season in league.get_seasons()[4:]: distance = {i: 0 for i in [0, 1, 2]} number_bet_odds = {i: 0 for i in [0, 1, 2]} print(season) for stage in range(1, league.get_stages_by_season(season) + 1): stage_predictions = predictor.predict(league, season, stage, **params) for match_id, pair in stage_predictions.items(): if len(pair) == 0: continue match = Match.read_by_match_id(match_id) if util.is_None(match.B365H) or util.is_None(match.B365D) or util.is_None(match.B365A)\ or match.B365H == 0 or match.B365D == 0 or match.B365A == 0: continue predicted_label, prob = pair[0], pair[1] our_bet_odds = 1 / prob bm_bet_odds = -1 if predicted_label == 1: bm_bet_odds = match.B365H elif predicted_label == 0: bm_bet_odds = match.B365D elif predicted_label == 2: bm_bet_odds = match.B365A distance[predicted_label] += math.fabs(our_bet_odds - bm_bet_odds) number_bet_odds[predicted_label] += 1 print(1, distance[1] / max(number_bet_odds[1], 1)) print(0, distance[0] / max(number_bet_odds[0], 1)) print(2, distance[2] / max(number_bet_odds[2], 1))
def look_for_matches(self, date, force_parsing=False): print("Elaborating matches of the date:", date) matches_link = self.host_url_match + "/sport_events/1%2F"+date+"%2Fbasic_h2h%2F0%2F0/" log.debug("Looking for matches of date ["+date+"] at link ["+matches_link+"]") try: page = requests.get(matches_link).text except Exception as e: print(e) soup = BeautifulSoup(page, "html.parser") header_list = soup.find_all('div', {'class': 'mx-default-header mx-text-align-left mx-flexbox-container '}) body_list = soup.find_all('div', {'class': 'mx-table mx-soccer mx-matches-table mx-group-by-stage mx-container mx-league mx-livescore-table'}) for header, body in zip(header_list, body_list): # reading the league # Notice that the league is identified also with an attribute called "data-stage" league_name = str(header.a.string).strip() league_data_stage = header.a.attrs['data-stage'] # check if the this league corresponds to one of those one managed! cl = CrawlerLeague(league_name, league_data_stage) if cl.is_in_a_managed_country() and len(league_name) > 3: league = cl.get_league() if util.is_None(league): log.warning("Impossible to crawl this league [" + league_name + ", " + league_data_stage + "]") continue print("\t- Looking for the league [" + league.name + "]") season = cl.get_season() for div_event in body.find_all('div', {'class': 'mx-stage-events'}): # event correspond to "match_api_id" event = str(div_event.attrs["class"][3]).split("-")[2] match = Match.read_by_match_api_id(event) if force_parsing \ or not match \ or not match.are_teams_linedup() \ or not match.are_incidents_managed() \ or not match.get_home_team() \ or not match.get_away_team(): # crawl when at least one of the following happen: # - match is not in the DB # - formation of the teams are not in the DB # - incidents of the match are not in the DB # - home_team_api_id is not matched to any team in the DB # - away_team_api_id is not matched to any team in the DB log.debug("Need to crawl match ["+event+"]") cm = CrawlerMatch(match, league, event) cm.parse_json(season) else: log.debug("Not need to crawl match [" + event + "]")
def read_players_api_id_by_team_api_id(team_api_id, season=None): """ return a list of player_api_id if season is set, consider only that list :param team_api_id: :param season: :return: """ players_api_id = set() filter = {} if season: filter["season"] = season else: season = "" try: return Cache.get_element( str(team_api_id) + "_" + season, "MATCH_GET_PLAYERS_BY_TEAM_API_ID") except KeyError: pass filter["home_team_api_id"] = team_api_id for sqllite_row in SQLLite.get_connection().select( "Match", column_filter="home_player_1, home_player_2, " "home_player_3, home_player_4, " "home_player_5, home_player_6, " "home_player_7, home_player_8, " "home_player_9, home_player_10, " "home_player_11", **filter): for home_player_i, player_api_id in sqllite_row.items(): if player_api_id: players_api_id.add(player_api_id) del (filter["home_team_api_id"]) filter["away_team_api_id"] = team_api_id for sqllite_row in SQLLite.get_connection().select( "Match", column_filter="away_player_1, away_player_2, " "away_player_3, away_player_4, " "away_player_5, away_player_6, " "away_player_7, away_player_8, " "away_player_9, away_player_10, " "away_player_11", **filter): for away_player_i, player_api_id in sqllite_row.items(): if not util.is_None(player_api_id): players_api_id.add(player_api_id) Cache.add_element( str(team_api_id) + "_" + season, players_api_id, "MATCH_GET_PLAYERS_BY_TEAM_API_ID") return players_api_id
def are_incidents_managed(self): """ Return TURE if and only if all the incidents are managed :return: """ if util.is_None(self.goal) \ or util.is_None(self.shoton) \ or util.is_None(self.shotoff) \ or util.is_None(self.foulcommit) \ or util.is_None(self.card) \ or util.is_None(self.cross) \ or util.is_None(self.corner) \ or util.is_None(self.possession): return False return True
def update(self, table_name, object): column_names = self.getColumnFromTable(table_name) update = "UPDATE " + table_name + " SET " for column in column_names: value = str(object.__getattribute__(column)).replace("'", "''") if util.is_None(value): continue update += column + "='" + value + "'," update = update[:-1] update += " WHERE id = '" + str(object.id) + "'" self.execute_update(update)
def read_by_team_api_id(team_api_id, season=None): """ Return list of players that play in the team identified my team_api_id if season is set, consider only that season :param team_api_id: :param season: :return: """ if not season: season = "" try: return Cache.get_element( str(team_api_id) + "_" + season, "PLAYER_BY_TEAM_API_ID") except KeyError: pass players = [] players_api_id = Match.read_players_api_id_by_team_api_id( team_api_id, season) for player_api_id in players_api_id: # if the player_api_id is not set --> continue if util.is_None(player_api_id): continue try: player = Cache.get_element(player_api_id, "PLAYER_BY_API_ID") except KeyError: filter = {"player_api_id": player_api_id} try: sqllite_row = SQLLite.get_connection().select( "Player", **filter)[0] except IndexError: log.warning("Player api id not found in DB [" + str(player_api_id) + "]") continue player = Player(sqllite_row["id"]) for attribute, value in sqllite_row.items(): player.__setattr__(attribute, value) Cache.add_element(player_api_id, player, "PLAYER_BY_API_ID") players.append(player) Cache.add_element( str(team_api_id) + "_" + season, players, "PLAYER_BY_TEAM_API_ID") return players
def get_matches(self, season=None, ordered=True, stage=None): """ Return the matches this player has played :param season: :param ordered: :param stage: :return: """ if util.is_None(self.player_api_id): return [] matches = Match.read_by_player_api_id(self.player_api_id) if season: matches = [m for m in matches if m.season == season] if stage: matches = [m for m in matches if m.stage < stage] if ordered: matches = sorted(matches, key=lambda match: match.stage) return matches
def get_goal_received(self, season=None, stage=None): """ Return the goal received by this player :param season: :param stage: :return: """ cnt = 0 current_team = self.get_current_team() matches = self.get_matches(season=season, ordered=True) for m in matches: if not util.is_None(stage) and m.stage >= stage: return cnt if m.home_team_api_id == current_team.team_api_id: cnt += m.away_team_goal else: cnt += m.home_team_goal return cnt
def get_formation(match, team, home=True): formation_str = "\n\n" + team.team_short_name # get trends formation_str += get_team_trend_str(match, team) # get Goals formation_str += get_team_goals_str(match, team) formation_str += get_team_goals_str(match, team, n=5) if home: team_lines = match.get_home_team_lines_up() else: team_lines = match.get_away_team_lines_up() team_goal_done, team_goal_received, num_matches = team.get_goals_by_season_and_stage( util.get_current_season(), match.stage) for i, player in enumerate(team_lines): if not util.is_None(player): if player.is_gk(): goal_dr = player.get_goal_received( season=util.get_current_season(), stage=match.stage) team_goal_dr = team_goal_received else: goal_dr = player.get_goal_done( season=util.get_current_season(), stage=match.stage) team_goal_dr = team_goal_done formation_str += '\n\t' + str(i + 1) + ") " + player.player_name.ljust(25, ".") \ + "\t P: "+str(len(player.get_matches(season=util.get_current_season(), stage=match.stage))) + '/' + str(match.stage-1) \ + "\t G: "+str(goal_dr)+"/"+str(team_goal_dr) if not player.is_gk(): assist_done = player.get_assist_done( season=util.get_current_season(), stage=match.stage) team_assist_done = team.get_assist_by_season_and_stage( season=util.get_current_season(), stage=match.stage) formation_str += "\t A: " + str(assist_done) + "/" + str( team_assist_done) return formation_str
def is_finished(self): """ finished if and only if incidents of the match are managed :return: """ is_scored_goals = self.home_team_goal > 0 or self.away_team_goal > 0 if (is_scored_goals and util.compare_time_to_now(self.date, 1)) \ or (not is_scored_goals and util.compare_time_to_now(self.date, 100)): return True if util.is_None(self.goal) \ and util.is_None(self.shoton) \ and util.is_None(self.shotoff) \ and util.is_None(self.foulcommit) \ and util.is_None(self.card) \ and util.is_None(self.cross) \ and util.is_None(self.corner) \ and util.is_None(self.possession): return False return True
def read_by_match_event_id_and_event_name(match_event_id, event_name=None): ''' Return the list of bet_event of the input match :param match_event_id: :param event_name: :return: ''' filter = {} filter["match_event_id"] = str(match_event_id) if not util.is_None(event_name): filter["event_name"] = event_name bet_events = [] for sqllite_row in SQLLite.get_connection().select("Bet_Event", **filter): be = BetEvent(sqllite_row["id"]) for attribute, value in sqllite_row.items(): be.__setattr__(attribute, value) bet_events.append(be) return bet_events
def get_matches(self, season=None, ordered=True, date=None, finished=None, stage=None): """ return the matches belonging to this league :param season: :param ordered: :param date: :param finished: :param stage: :return: """ matches = Match.read_matches_by_league(self.id, season) if ordered: try: matches = sorted(matches, key=lambda match: match.stage) except TypeError as e: log.error("Impossible to order match of the league [" + str(self.id) + "], of the season [" + season + "] by stage") raise e if not util.is_None(finished) and finished: matches = [m for m in matches if m.is_finished()] if stage: matches = [m for m in matches if m.stage == stage] if date: matches = [m for m in matches if m.date.startswith(date)] return matches
def get_training_ranking(self, season, stage_to_predict, stages_to_train, home=None): """ :param season: :param stage_to_predict: :param stages_to_train: :param home: :return: """ import src.application.Domain.Team as Team matches = self.get_training_matches(season, stage_to_predict, stages_to_train) teams = self.get_teams(season=season) ranking = {team.id: 0 for team in teams} num_matches = {team.id: 0 for team in teams} for m in matches: try: result, winner = m.get_winner() if result == 0: winner_id = None else: winner_id = winner.id home_id = m.get_home_team().id away_id = m.get_away_team().id except AttributeError: continue if util.is_None(home): # total ranking util.increase_dict_entry(home_id, num_matches) util.increase_dict_entry(away_id, num_matches) if result != 0: util.increase_dict_entry(winner_id, ranking, 3) else: util.increase_dict_entry(home_id, ranking, 1) util.increase_dict_entry(away_id, ranking, 1) elif home: # home ranking util.increase_dict_entry(home_id, num_matches) if result == 1: util.increase_dict_entry(winner_id, ranking, 3) elif result == 0: util.increase_dict_entry(home_id, ranking, 1) else: # away ranking util.increase_dict_entry(away_id, num_matches) if result == 2: util.increase_dict_entry(winner_id, ranking, 3) elif result == 0: util.increase_dict_entry(away_id, ranking, 1) # divide the overall point by the number of matches done norm_ranking = dict() for team_id, points in ranking.items(): if num_matches[team_id] > 0: norm_ranking[team_id] = points / num_matches[team_id] else: norm_ranking[team_id] = 0 # order the normalized rank ranking_ret = [] for team, p in sorted(norm_ranking.items(), key=operator.itemgetter(1))[::-1]: ranking_ret.append((p, Team.read_by_id(team))) return ranking_ret
def get_training_matches(self, season, stage_to_predict, stages_to_train, consider_last=False): """ Return the training matches needed by the machine learning algorithm :param season: :param stage_to_predict: :param stages_to_train: :param consider_last: :return: """ if util.is_None(stages_to_train): # stages to train not defined --> return only stage of this season return [ m for m in self.get_matches( season=season, ordered=True, finished=True) if m.stage < stage_to_predict ] else: # stages to train is defined --> return all the matches played it those number of stages # EX: stage_to_predict = 7, n_match_1_stage = 10 --> return 70 matches if consider_last: # start to consider matches of previous seasons --> take them in the reverse order training_matches = [ m for m in self.get_matches( season=season, ordered=True, finished=True) ] training_matches = training_matches[::-1] else: training_matches = [ m for m in self.get_matches( season=season, ordered=True, finished=True) if m.stage < stage_to_predict ] if len(training_matches) == 0 and season < '2006/2007': raise MLException(0) stages_training = set([(m.stage, m.season) for m in training_matches]) if len(stages_training) < stages_to_train: # need more matches from the past season, considering the last matches past_training_matches = self.get_training_matches( util.get_previous_season(season), 0, stages_to_train - len(stages_training), consider_last=True) training_matches.extend(past_training_matches) n_matches_in_stage = int(len(self.get_teams_current_season()) / 2) if len(training_matches) / n_matches_in_stage > stages_to_train: # too matches in training --> remove too far if consider_last: return training_matches[:stages_to_train * n_matches_in_stage][::-1] else: return training_matches[-stages_to_train * n_matches_in_stage:] return training_matches