def fill_results_from_api(gw_start, gw_end, season, dbsession):
fetcher = FPLDataFetcher()
matches = fetcher.get_fixture_data()
for m in matches:
if not m["finished"]:
continue
gameweek = m["event"]
if gameweek < gw_start or gameweek > gw_end:
continue
home_id = m["team_h"]
away_id = m["team_a"]
home_team = None
away_team = None
for k, v in alternative_team_names.items():
if str(home_id) in v:
home_team = k
elif str(away_id) in v:
away_team = k
if not home_team:
raise ValueError("Unable to find team with id {}".format(home_id))
if not away_team:
raise ValueError("Unable to find team with id {}".format(away_id))
home_score = m["team_h_score"]
away_score = m["team_a_score"]
f = _find_fixture(season, home_team, away_team, dbsession)
res = Result()
res.fixture = f
res.home_score = int(home_score)
res.away_score = int(away_score)
dbsession.add(res)
dbsession.commit()
def main():
parser = argparse.ArgumentParser(description="plot mini-league")
parser.add_argument(
"--thing_to_plot",
help="points,total_points,ranking,overall_ranking",
default="total_points",
)
args = parser.parse_args()
thing_to_plot = args.thing_to_plot
fetcher = FPLDataFetcher()
league_data = fetcher.get_fpl_league_data()
team_ids = get_team_ids(league_data)
team_names = get_team_names(league_data)
team_histories = []
for i, team_id in enumerate(team_ids):
team_data = fetcher.get_fpl_team_history_data(team_id)
history_dict = get_team_history(team_data)
history_dict["name"] = team_names[i]
team_histories.append(history_dict)
xvals = sorted(team_histories[0]["history"].keys())
points = []
for th in team_histories:
points.append([
th["history"][gw][thing_to_plot]
for gw in sorted(th["history"].keys())
])
plt.plot(xvals, points[-1], label=th["name"])
plt.legend(loc="best")
plt.xlabel("gameweek")
plt.ylabel(thing_to_plot)
plt.show()
def test_get_summary_data():
"""
get summary of all players' data for this season.
"""
fetcher = FPLDataFetcher()
data = fetcher.get_current_summary_data()
assert isinstance(data, dict)
assert len(data) > 0
def test_get_event_data():
"""
gameweek list with deadlines and status
"""
fetcher = FPLDataFetcher()
data = fetcher.get_event_data()
assert isinstance(data, dict)
assert len(data) > 0
def test_get_team_history_data():
"""
gameweek history for our team id
"""
fetcher = FPLDataFetcher()
data = fetcher.get_fpl_team_history_data()
assert isinstance(data, dict)
assert len(data) > 0
def test_get_player_summary_data():
"""
summary for individual players
"""
fetcher = FPLDataFetcher()
data = fetcher.get_player_summary_data()
assert isinstance(data, dict)
assert len(data) > 0
def test_get_current_team_data():
"""
summary for current teams
"""
fetcher = FPLDataFetcher()
data = fetcher.get_current_team_data()
assert isinstance(data, dict)
assert len(data) > 0
def test_get_team_data():
"""
should give current list of players in our team
"""
fetcher = FPLDataFetcher()
data = fetcher.get_fpl_team_data(1)["picks"]
assert isinstance(data, list)
assert len(data) == 15
def test_get_detailed_player_data():
"""
for player_id=1, list of gameweek data
"""
fetcher = FPLDataFetcher()
data = fetcher.get_gameweek_data_for_player(1)
assert isinstance(data, dict)
assert len(data) > 0
def test_get_fpl_team_data_gw1():
"""
which players are in our squad for gw1
"""
fetcher = FPLDataFetcher()
data = fetcher.get_fpl_team_data(1)
assert isinstance(data, dict)
assert "picks" in data.keys()
players = [p["element"] for p in data["picks"]]
assert len(players) == 15
def fill_fixtures_from_api(season, dbsession=session):
"""
Use the FPL API to get a list of fixures.
"""
tag = str(uuid.uuid4())
fetcher = FPLDataFetcher()
fixtures = fetcher.get_fixture_data()
for fixture in fixtures:
try:
f = find_fixture(
fixture["team_h"],
was_home=True,
other_team=fixture["team_a"],
season=season,
dbsession=dbsession,
)
update = True
except ValueError:
f = Fixture()
update = False
f.date = fixture["kickoff_time"]
f.gameweek = fixture["event"]
f.season = season
f.tag = tag
home_id = fixture["team_h"]
away_id = fixture["team_a"]
found_home = False
found_away = False
for k, v in alternative_team_names.items():
if str(home_id) in v:
f.home_team = k
found_home = True
elif str(away_id) in v:
f.away_team = k
found_away = True
if found_home and found_away:
break
error_str = "Can't find team(s) with id(s): {}."
if not found_home and found_away:
raise ValueError(error_str.format(home_id + ", " + away_id))
elif not found_home:
raise ValueError(error_str.format(home_id))
elif not found_away:
raise ValueError(error_str.format(away_id))
else:
pass
if not update:
dbsession.add(f)
dbsession.commit()
return True
def make_transfers(fpl_team_id=None, skip_check=False):
transfer_player_ids, team_id, current_gw, chip_played = get_gw_transfer_suggestions(
fpl_team_id
)
fetcher = FPLDataFetcher(team_id)
if len(transfer_player_ids[0]) == 0:
# no players to remove in DB - initial team?
print("Making transfer list for starting team")
priced_transfers = build_init_priced_transfers(fetcher, team_id)
else:
pre_transfer_bank = get_bank(fpl_team_id=team_id)
priced_transfers = price_transfers(transfer_player_ids, fetcher, current_gw)
post_transfer_bank = deduct_transfer_price(pre_transfer_bank, priced_transfers)
print_output(
team_id, current_gw, priced_transfers, pre_transfer_bank, post_transfer_bank
)
if skip_check or check_proceed():
transfer_req = build_transfer_payload(
priced_transfers, current_gw, fetcher, chip_played
)
post_transfers(transfer_req, fetcher)
return True
def test_get_fpl_team_data_gw1_different_fpl_team_ids():
"""
which players are in a couple of different squads for gw 1
"""
fetcher = FPLDataFetcher()
# assume that fpl_team_ids < 100 will all have squads for
# gameweek 1, and that they will be different..
team_id_1 = random.randint(1, 50)
team_id_2 = random.randint(51, 100)
data_1 = fetcher.get_fpl_team_data(1, fpl_team_id=team_id_1)
players_1 = [p["element"] for p in data_1["picks"]]
assert len(players_1) == 15
data_2 = fetcher.get_fpl_team_data(1, fpl_team_id=team_id_2)
players_2 = [p["element"] for p in data_2["picks"]]
assert len(players_2) == 15
# check they are different
assert sorted(players_1) != sorted(players_2)
def fill_player_table_from_api(season, session):
"""
use the FPL API
"""
df = FPLDataFetcher()
pd = df.get_player_summary_data()
for k, v in pd.items():
p = Player()
p.fpl_api_id = k
first_name = v["first_name"] # .encode("utf-8")
second_name = v["second_name"] # .encode("utf-8")
name = "{} {}".format(first_name, second_name)
print("PLAYER {} {}".format(season, name))
p.name = name
session.add(p)
session.commit()
def set_lineup(fpl_team_id=None):
"""
Retrieve the latest lineup and apply the latest prediction to it.
Note that this assumes that the prediction has been ran recently.
"""
print("fpl_team_id is {}".format(fpl_team_id))
fetcher = FPLDataFetcher(fpl_team_id)
print("Got fetcher {}".format(fetcher.FPL_TEAM_ID))
picks = fetcher.get_lineup()
print("Got picks {}".format(picks))
squad = get_lineup_from_payload(picks)
print("got squad: {}".format(squad))
squad.optimize_lineup(NEXT_GAMEWEEK, get_latest_prediction_tag())
if check_proceed(squad):
payload = build_lineup_payload(squad)
fetcher.post_lineup(payload)
def main(fpl_team_id=None):
transfer_player_ids, team_id, current_gw, chip_played = get_gw_transfer_suggestions(
fpl_team_id
)
fetcher = FPLDataFetcher(team_id)
pre_transfer_bank = get_bank(fpl_team_id=team_id)
priced_transfers = price_transfers(transfer_player_ids, fetcher, current_gw)
post_transfer_bank = deduct_transfer_price(pre_transfer_bank, priced_transfers)
print_output(
team_id, current_gw, priced_transfers, pre_transfer_bank, post_transfer_bank
)
if check_proceed():
transfer_req = build_transfer_payload(
priced_transfers, current_gw, fetcher, chip_played
)
post_transfers(transfer_req, fetcher)
def fill_playerscores_from_api(season,
gw_start=1,
gw_end=NEXT_GAMEWEEK,
dbsession=session):
fetcher = FPLDataFetcher()
input_data = fetcher.get_player_summary_data()
for player_api_id in input_data.keys():
# find the player in the player table. If they're not
# there, then we don't care (probably not a current player).
player = get_player_from_api_id(player_api_id, dbsession=dbsession)
if not player:
print("No player with API id {}".format(player_api_id))
player_id = player.player_id
print("SCORES {} {}".format(season, player.name))
player_data = fetcher.get_gameweek_data_for_player(player_api_id)
# now loop through all the matches that player played in
for gameweek, results in player_data.items():
if gameweek not in range(gw_start, gw_end):
continue
for result in results:
# try to find the match in the match table
opponent = get_team_name(result["opponent_team"])
played_for, fixture = get_player_team_from_fixture(
gameweek,
opponent,
player_at_home=result["was_home"],
kickoff_time=result["kickoff_time"],
season=season,
dbsession=dbsession,
return_fixture=True,
)
if not fixture or not played_for:
print(" Couldn't find match for {} in gw {}".format(
player.name, gameweek))
continue
ps = PlayerScore()
ps.player_team = played_for
ps.opponent = opponent
ps.goals = result["goals_scored"]
ps.assists = result["assists"]
ps.bonus = result["bonus"]
ps.points = result["total_points"]
ps.conceded = result["goals_conceded"]
ps.minutes = result["minutes"]
ps.player = player
ps.fixture = fixture
ps.result = fixture.result
# extended features
# get features excluding the core ones already populated above
extended_feats = [
col for col in ps.__table__.columns.keys() if col not in [
"id",
"player_team",
"opponent",
"goals",
"assists",
"bonus",
"points",
"conceded",
"minutes",
"player_id",
"result_id",
"fixture_id",
]
]
for feat in extended_feats:
try:
ps.__setattr__(feat, result[feat])
except KeyError:
pass
dbsession.add(ps)
print(" got {} points vs {} in gameweek {}".format(
result["total_points"], opponent, gameweek))
def fill_attributes_table_from_api(season, gw_start=1, dbsession=session):
"""
use the FPL API to get player attributes info for the current season
"""
fetcher = FPLDataFetcher()
next_gw = get_next_gameweek(season=season, dbsession=dbsession)
# needed for selected by calculation from percentage below
n_players = fetcher.get_current_summary_data()["total_players"]
input_data = fetcher.get_player_summary_data()
for player_api_id in input_data.keys():
# find the player in the player table
player = get_player_from_api_id(player_api_id, dbsession=dbsession)
if not player:
print("ATTRIBUTES {} No player found with id {}".format(
season, player_api_id))
continue
print("ATTRIBUTES {} {}".format(season, player.name))
# First update the current gameweek using the summary data
p_summary = input_data[player_api_id]
position = positions[p_summary["element_type"]]
pa = get_player_attributes(player.player_id,
season=season,
gameweek=next_gw,
dbsession=dbsession)
if pa:
# found pre-existing attributes for this gameweek
update = True
else:
# no attributes for this gameweek for this player yet
pa = PlayerAttributes()
update = False
pa.player = player
pa.player_id = player.player_id
pa.season = season
pa.gameweek = next_gw
pa.price = int(p_summary["now_cost"])
pa.team = get_team_name(p_summary["team"],
season=season,
dbsession=dbsession)
pa.position = positions[p_summary["element_type"]]
pa.selected = int(
float(p_summary["selected_by_percent"]) * n_players / 100)
pa.transfers_in = int(p_summary["transfers_in_event"])
pa.transfers_out = int(p_summary["transfers_out_event"])
pa.transfers_balance = pa.transfers_in - pa.transfers_out
pa.chance_of_playing_next_round = p_summary[
"chance_of_playing_next_round"]
pa.news = p_summary["news"]
if (pa.chance_of_playing_next_round is not None
and pa.chance_of_playing_next_round <= 50):
pa.return_gameweek = get_return_gameweek_from_news(
p_summary["news"],
season=season,
dbsession=dbsession,
)
if not update:
# only need to add to the dbsession for new entries, if we're doing
# an update the final dbsession.commit() is enough
dbsession.add(pa)
# now get data for previous gameweeks
if next_gw > 1:
player_data = fetcher.get_gameweek_data_for_player(player_api_id)
if not player_data:
print("Failed to get data for", player.name)
continue
for gameweek, data in player_data.items():
if gameweek < gw_start:
continue
for result in data:
# check whether there are pre-existing attributes to update
pa = get_player_attributes(
player.player_id,
season=season,
gameweek=gameweek,
dbsession=dbsession,
)
if pa:
update = True
else:
pa = PlayerAttributes()
update = False
# determine the team the player played for in this fixture
opponent_id = result["opponent_team"]
was_home = result["was_home"]
kickoff_time = result["kickoff_time"]
team = get_player_team_from_fixture(
gameweek,
opponent_id,
was_home,
kickoff_time,
season=season,
dbsession=dbsession,
)
pa.player = player
pa.player_id = player.player_id
pa.season = season
pa.gameweek = gameweek
pa.price = int(result["value"])
pa.team = team
pa.position = position # does not change during season
pa.transfers_balance = int(result["transfers_balance"])
pa.selected = int(result["selected"])
pa.transfers_in = int(result["transfers_in"])
pa.transfers_out = int(result["transfers_out"])
if not update:
# don't need to add to dbsession if updating pre-existing row
dbsession.add(pa)
break # done this gameweek now
names
"""
best_ratio = 0.0
best_match = None
for t in fpl_teams:
if fuzz.partial_ratio(t, team) > best_ratio:
best_ratio = fuzz.partial_ratio(t, team)
best_match = t
print("Best match {}/{}, score {}".format(best_match, team, best_ratio))
return best_match, best_ratio
if __name__ == "__main__":
# get the team names as used in FPL
df = FPLDataFetcher()
teamdata = df.get_current_team_data()
teamdict = {
teamdata[k]["name"]: [teamdata[k]["short_name"]]
for k in teamdata.keys()
}
# teamdicts = [{teamdata[k]['name']:[teamdata[k]['short_name']]} \
# for k in teamdata.keys()]
fpl_teams = list(teamdict.keys())
# get the team names from the results csv
missing = set()
matched = set()
history_teams = set()
for season in ["1415", "1516", "1617", "1718"]:
filename = "../data/results_{}.csv".format(season)
"""
Season details
"""
from datetime import datetime
from airsenal.framework.data_fetcher import FPLDataFetcher
def get_current_season():
"""
use the current time to find what season we're in.
"""
current_time = datetime.now()
if current_time.month > 7:
start_year = current_time.year
else:
start_year = current_time.year - 1
end_year = start_year + 1
return "{}{}".format(str(start_year)[2:], str(end_year)[2:])
# make this a global variable in this module, import into other modules
CURRENT_SEASON = get_current_season()
fetcher = FPLDataFetcher()
# TODO make this a database table so we can look at past seasons
CURRENT_TEAMS = [
t["short_name"] for t in fetcher.get_current_team_data().values()
]
def fill_playerscores_from_api(season,
gw_start=1,
gw_end=NEXT_GAMEWEEK,
dbsession=session):
fetcher = FPLDataFetcher()
input_data = fetcher.get_player_summary_data()
for player_api_id in input_data.keys():
player = get_player_from_api_id(player_api_id, dbsession=dbsession)
if not player:
# If no player found with this API ID something has gone wrong with the
# Player table, e.g. clashes between players with the same name
print(f"ERROR! No player with API id {player_api_id}. Skipped.")
continue
print("SCORES {} {}".format(season, player))
player_data = fetcher.get_gameweek_data_for_player(player_api_id)
# now loop through all the matches that player played in
for gameweek, results in player_data.items():
if gameweek not in range(gw_start, gw_end):
continue
for result in results:
# try to find the match in the match table
opponent = get_team_name(result["opponent_team"])
played_for, fixture = get_player_team_from_fixture(
gameweek,
opponent,
player_at_home=result["was_home"],
kickoff_time=result["kickoff_time"],
season=season,
dbsession=dbsession,
return_fixture=True,
)
if not fixture or not played_for or not fixture.result:
print(
" Couldn't find match result for {} in gw {}".format(
player, gameweek))
continue
ps = get_player_scores(fixture=fixture,
player=player,
dbsession=dbsession)
if ps is None:
ps = PlayerScore()
add = True
else:
add = False
ps.player_team = played_for
ps.opponent = opponent
ps.goals = result["goals_scored"]
ps.assists = result["assists"]
ps.bonus = result["bonus"]
ps.points = result["total_points"]
ps.conceded = result["goals_conceded"]
ps.minutes = result["minutes"]
ps.player = player
ps.fixture = fixture
ps.result = fixture.result
# extended features
# get features excluding the core ones already populated above
extended_feats = [
col for col in ps.__table__.columns.keys() if col not in [
"id",
"player_team",
"opponent",
"goals",
"assists",
"bonus",
"points",
"conceded",
"minutes",
"player_id",
"result_id",
"fixture_id",
]
]
for feat in extended_feats:
try:
ps.__setattr__(feat, result[feat])
except KeyError:
pass
if add:
dbsession.add(ps)
print(" got {} points vs {} in gameweek {}".format(
result["total_points"], opponent, gameweek))
dbsession.commit()
best_ratio = 0.0
best_match = None
for p in fpl_players:
if fuzz.partial_ratio(p, player) > best_ratio:
best_ratio = fuzz.partial_ratio(p, player)
best_match = p
# print("Best match {}/{}, score {}".format(best_match,
# player,
# best_ratio))
return best_match, best_ratio
if __name__ == "__main__":
# get the team names as used in FPL
df = FPLDataFetcher()
playerdict = {}
playerdata = df.get_player_summary_data()
for k in playerdata.keys():
player_name = "{} {}".format(playerdata[k]["first_name"],
playerdata[k]["second_name"])
playerdict[player_name] = [playerdata[k]["web_name"]]
fpl_players_to_match = list(playerdict.keys())
# get the player names from the fpl archives json
missing = set()
matched = set()
history_players = set()
for season in ["1516", "1617"]:
filename = "../data/player_summary_{}.json".format(season)
player_data = json.load(open(filename))
from airsenal.framework.mappings import alternative_player_names
from airsenal.framework.data_fetcher import FPLDataFetcher
from airsenal.framework.schema import (
Player,
PlayerAttributes,
Fixture,
PlayerScore,
PlayerPrediction,
Transaction,
Team,
session,
)
from airsenal.framework.season import CURRENT_SEASON
fetcher = FPLDataFetcher() # in global scope so it can keep cached data
def get_max_gameweek(season=CURRENT_SEASON, dbsession=session):
"""
Return the maximum gameweek number across all scheduled fixtures. This shuold
generally be 38, but may be different in the case of major disruptino (e.g.
Covid-19)
"""
max_gw_fixture = (dbsession.query(Fixture).filter_by(
season=season).order_by(Fixture.gameweek.desc()).first())
if max_gw_fixture is None:
# TODO Tests fail without this as tests don't populate fixture table in db
max_gw = 100
else:
def test_instantiate_fetchers():
"""
check we can instantiate the classes
"""
fpl = FPLDataFetcher()
assert fpl
