def to_dict(t):
return {
"element_out": get_player(t[0][0]).fpl_api_id,
"selling_price": t[0][1],
"element_in": get_player(t[1][0]).fpl_api_id,
"purchase_price": t[1][1],
}
def get_sell_price_for_player(
self,
player,
use_api=False,
gameweek=NEXT_GAMEWEEK,
dbsession=None,
apifetcher=fetcher,
):
"""Get sale price for player (a player in self.players) in the current
gameweek of the current season.
"""
player_id = player.player_id
price_now = None
if use_api and self.season == CURRENT_SEASON and gameweek >= NEXT_GAMEWEEK:
try:
# first try getting the price for the player from the API
player_db = get_player(player_id)
api_id = player_db.fpl_api_id
if (apifetcher.logged_in and apifetcher.FPL_TEAM_ID
and apifetcher.FPL_TEAM_ID != "MISSING_ID"):
return apifetcher.get_current_picks(
)[api_id]["selling_price"]
# if not logged in, just get current price from API
price_now = apifetcher.get_player_summary_data(
)[api_id]["now_cost"]
except Exception:
pass
# retrieve how much we originally bought the player for from db
price_bought = player.purchase_price
if not price_now:
player_db = get_player(player_id, dbsession=dbsession)
if player_db:
price_now = player_db.price(self.season, gameweek)
if not price_now:
# if all else fails just use the purchase price as the sale
# price for this player.
print(
"Using purchase price as sale price for",
player.player_id,
player,
)
price_now = price_bought
if price_now > price_bought:
return (price_now + price_bought) // 2
else:
return price_now
def to_dict(player, pos_int):
return {
"element": get_player(player.player_id).fpl_api_id,
"position": pos_int,
"is_captain": player.is_captain,
"is_vice_captain": player.is_vice_captain,
}
def combine_player_info(player_id, dbsession=DBSESSION):
"""
Get player's name, club, recent scores, upcoming fixtures, and
upcoming predictions if available
"""
info_dict = {"player_id": player_id}
p = get_player(player_id, dbsession=dbsession)
info_dict["player_name"] = p.name
team = p.team(CURRENT_SEASON, NEXT_GAMEWEEK)
info_dict["team"] = team
# get recent scores for the player
rs = get_recent_scores_for_player(p, dbsession=dbsession)
recent_scores = [{"gameweek": k, "score": v} for k, v in rs.items()]
info_dict["recent_scores"] = recent_scores
# get upcoming fixtures
fixtures = get_fixtures_for_player(p, dbsession=dbsession)[:3]
info_dict["fixtures"] = []
for f in fixtures:
home_or_away = "home" if f.home_team == team else "away"
opponent = f.away_team if home_or_away == "home" else f.home_team
info_dict["fixtures"].append({
"gameweek": f.gameweek,
"opponent": opponent,
"home_or_away": home_or_away
})
try:
tag = get_latest_prediction_tag(dbsession=dbsession)
predicted_points = get_predicted_points_for_player(p,
tag,
dbsession=dbsession)
info_dict["predictions"] = predicted_points
except (RuntimeError):
pass
return info_dict
def price_transfers(transfer_player_ids, fetcher, current_gw):
"""
For most gameweeks, we get transfer suggestions from the db, including
both players to be removed and added.
"""
transfers = list(zip(*transfer_player_ids)) # [(out,in),(out,in)]
priced_transfers = [
[
[t[0], get_sell_price(fetcher.FPL_TEAM_ID, t[0])],
[
t[1],
fetcher.get_player_summary_data()[get_player(t[1]).fpl_api_id][
"now_cost"
],
],
]
for t in transfers
]
def to_dict(t):
return {
"element_out": get_player(t[0][0]).fpl_api_id,
"selling_price": t[0][1],
"element_in": get_player(t[1][0]).fpl_api_id,
"purchase_price": t[1][1],
}
transfer_list = [to_dict(transfer) for transfer in priced_transfers]
return transfer_list
def add_players_to_db(players_from_db, players_from_api, player_data_from_api,
dbsession):
print("Updating player table...")
# find the new player(s) from the API
api_ids_from_db = [p.fpl_api_id for p in players_from_db]
new_players = [p for p in players_from_api if p not in api_ids_from_db]
for player_api_id in new_players:
first_name = player_data_from_api[player_api_id]["first_name"]
second_name = player_data_from_api[player_api_id]["second_name"]
name = "{} {}".format(first_name, second_name)
# check whether we alreeady have this player in the database -
# if yes update that player's data, if no create a new player
p = get_player(name, dbsession=dbsession)
if p is None:
print("Adding player {}".format(name))
p = Player()
update = False
elif p.fpl_api_id is None:
print("Updating player {}".format(name))
update = True
else:
update = True
p.fpl_api_id = player_api_id
p.name = name
if not update:
dbsession.add(p)
dbsession.commit()
return len(new_players)
def fill_initial_team(session,
season=CURRENT_SEASON,
tag="AIrsenal" + CURRENT_SEASON):
"""
Fill the Transactions table in the database with the initial 15 players, and their costs,
getting the information from the team history API endpoint (for the list of players in our team)
and the player history API endpoint (for their price in gw1).
"""
print("SQUAD Getting selected players for gameweek 1...")
if NEXT_GAMEWEEK == 1:
### Season hasn't started yet - there won't be a team in the DB
return True
free_hit = free_hit_used_in_gameweek(1)
init_players = get_players_for_gameweek(1)
for pid in init_players:
player_api_id = get_player(pid).fpl_api_id
gw1_data = fetcher.get_gameweek_data_for_player(player_api_id, 1)
if len(gw1_data) == 0:
# Edge case where API doesn't have player data for gameweek 1, e.g. in 20/21
# season where 4 teams didn't play gameweek 1. Calculate GW1 price from
# API using current price and total price change.
print(
"Using current data to determine starting price for player {}".
format(player_api_id))
pdata = fetcher.get_player_summary_data()[player_api_id]
price = pdata["now_cost"] - pdata["cost_change_start"]
else:
price = gw1_data[0]["value"]
add_transaction(pid, 1, 1, price, season, tag, free_hit, session)
def test_get_player(fill_players):
"""
test we can get a player object from either a name or an id
"""
with test_session_scope() as tsession:
p = get_player("Bob", tsession)
assert isinstance(p, Player)
assert p.player_id == 1
def fill_initial_squad(
season=CURRENT_SEASON,
tag="AIrsenal" + CURRENT_SEASON,
fpl_team_id=None,
dbsession=session,
):
"""
Fill the Transactions table in the database with the initial 15 players, and their
costs, getting the information from the team history API endpoint (for the list of
players in our team) and the player history API endpoint (for their price in gw1).
"""
if not fpl_team_id:
fpl_team_id = fetcher.FPL_TEAM_ID
print(
"Getting initially selected players in squad {} for first gameweek...".
format(fpl_team_id))
if NEXT_GAMEWEEK == 1:
# Season hasn't started yet - there won't be a team in the DB
return True
starting_gw = get_entry_start_gameweek(fpl_team_id)
print(
f"Got starting squad from gameweek {starting_gw}. Adding player data..."
)
init_players = get_players_for_gameweek(starting_gw, fpl_team_id)
free_hit = free_hit_used_in_gameweek(starting_gw, fpl_team_id)
time = fetcher.get_event_data()[starting_gw]["deadline"]
for pid in init_players:
player_api_id = get_player(pid).fpl_api_id
first_gw_data = fetcher.get_gameweek_data_for_player(
player_api_id, starting_gw)
if len(first_gw_data) == 0:
# Edge case where API doesn't have player data for gameweek 1, e.g. in 20/21
# season where 4 teams didn't play gameweek 1. Calculate GW1 price from
# API using current price and total price change.
print(
"Using current data to determine starting price for player {}".
format(player_api_id))
pdata = fetcher.get_player_summary_data()[player_api_id]
price = pdata["now_cost"] - pdata["cost_change_start"]
else:
price = first_gw_data[0]["value"]
add_transaction(
pid,
starting_gw,
1,
price,
season,
tag,
free_hit,
fpl_team_id,
time,
dbsession,
)
def get_sell_price_for_player(
self,
player,
use_api=False,
season=CURRENT_SEASON,
gameweek=NEXT_GAMEWEEK,
dbsession=None,
):
"""Get sale price for player (a player in self.players) in the current
gameweek of the current season.
"""
price_bought = player.purchase_price
player_id = player.player_id
price_now = None
if use_api and season == CURRENT_SEASON and gameweek >= NEXT_GAMEWEEK:
try:
# first try getting the price for the player from the API
player_db = get_player(player_id)
price_now = fetcher.get_player_summary_data()[
player_db.fpl_api_id]["now_cost"]
except:
pass
if not price_now:
player_db = get_player(player_id, dbsession=dbsession)
if player_db:
price_now = player_db.price(season, gameweek)
if not price_now:
# if all else fails just use the purchase price as the sale
# price for this player.
print(
"Using purchase price as sale price for",
player.player_id,
player.name,
)
price_now = price_bought
if price_now > price_bought:
price_sell = (price_now + price_bought) // 2
else:
price_sell = price_now
return price_sell
def update_players(season, dbsession):
"""
See if any new players have been added to FPL since we last filled the 'player'
table in the db. If so, add them.
"""
players_from_db = list_players(
position="all", team="all", season=season, dbsession=dbsession
)
player_data_from_api = fetcher.get_player_summary_data()
players_from_api = list(player_data_from_api.keys())
if len(players_from_db) == len(players_from_api):
print("Player table already up-to-date.")
return 0
elif len(players_from_db) > len(players_from_api):
raise RuntimeError(
"Something strange has happened - more players in DB than API"
)
else:
print("Updating player table...")
# find the new player(s) from the API
api_ids_from_db = [p.fpl_api_id for p in players_from_db]
new_players = [p for p in players_from_api if p not in api_ids_from_db]
for player_api_id in new_players:
first_name = player_data_from_api[player_api_id]["first_name"]
second_name = player_data_from_api[player_api_id]["second_name"]
name = "{} {}".format(first_name, second_name)
# check whether we alreeady have this player in the database -
# if yes update that player's data, if no create a new player
p = get_player(name, dbsession=dbsession)
if p is None:
print("Adding player {}".format(name))
p = Player()
update = False
elif p.fpl_api_id is None:
print("Updating player {}".format(name))
update = True
else:
update = True
p.fpl_api_id = player_api_id
p.name = name
if not update:
dbsession.add(p)
dbsession.commit()
return len(new_players)
def price_transfers(transfer_player_ids, fetcher, current_gw):
transfers = list(zip(*transfer_player_ids)) # [(out,in),(out,in)]
# [[[out, price], [in, price]],[[out,price],[in,price]]]
priced_transfers = [
[
[t[0], get_sell_price(fetcher.FPL_TEAM_ID, t[0])],
[
t[1],
fetcher.get_player_summary_data()[get_player(t[1]).fpl_api_id][
"now_cost"
],
],
]
for t in transfers
]
return priced_transfers
def __init__(
self, player, season=CURRENT_SEASON, gameweek=NEXT_GAMEWEEK, dbsession=None
):
"""
initialize either by name or by ID
"""
self.dbsession = dbsession
if isinstance(player, Player):
pdata = player
else:
pdata = get_player(player, self.dbsession)
self.player_id = pdata.player_id
self.name = pdata.name
self.team = pdata.team(season, gameweek)
self.position = pdata.position(season)
self.purchase_price = pdata.price(season, gameweek)
self.is_starting = True # by default
self.is_captain = False # by default
self.is_vice_captain = False # by default
self.predicted_points = {}
self.sub_position = None
def fill_attributes_table_from_file(detail_data, season, session):
"""Fill player attributes table for previous season using data from
player detail JSON files.
"""
for player_name in detail_data.keys():
# find the player id in the player table. If they're not
# there, then we don't care (probably not a current player).
player = get_player(player_name, dbsession=session)
if not player:
print("Couldn't find player {}".format(player_name))
continue
print("ATTRIBUTES {} {}".format(season, player.name))
# now loop through all the fixtures that player played in
# Only one attributes row per gameweek - create list of gameweeks
# encountered so can ignore duplicates (e.g. from double gameweeks).
previous_gameweeks = []
for fixture_data in detail_data[player_name]:
gameweek = int(fixture_data["gameweek"])
if gameweek in previous_gameweeks:
# already done this gameweek
continue
else:
previous_gameweeks.append(gameweek)
pa = PlayerAttributes()
pa.player = player
pa.player_id = player.player_id
pa.season = season
pa.gameweek = gameweek
pa.price = int(fixture_data["value"])
pa.team = fixture_data["played_for"]
pa.position = fixture_data["position"]
pa.transfers_balance = int(fixture_data["transfers_balance"])
pa.selected = int(fixture_data["selected"])
pa.transfers_in = int(fixture_data["transfers_in"])
pa.transfers_out = int(fixture_data["transfers_out"])
session.add(pa)
def build_init_priced_transfers(fetcher, fpl_team_id=None):
"""
Before gameweek 1, there won't be any 'sell' transfer suggestions in the db.
We can instead query the API for our current 'picks' (requires login).
"""
if not fpl_team_id:
if (not fetcher.FPL_TEAM_ID) or fetcher.FPL_TEAM_ID == "MISSING_ID":
fpl_team_id = int(input("Please enter FPL team ID: "))
else:
fpl_team_id = fetcher.FPL_TEAM_ID
current_squad = fetcher.get_current_picks(fpl_team_id)
transfers_out = [
{"element_out": el["element"], "selling_price": el["selling_price"]}
for el in current_squad
]
transfer_in_suggestions = get_transfer_suggestions(dbsession)
if len(transfers_out) != len(transfer_in_suggestions):
raise RuntimeError(
"Number of transfers in and out don't match: {} {}".format(
len(transfer_in_suggestions), len(transfers_out)
)
)
transfers_in = []
for t in transfer_in_suggestions:
api_id = get_player(t.player_id).fpl_api_id
price = fetcher.get_player_summary_data()[api_id]["now_cost"]
transfers_in.append({"element_in": api_id, "purchase_price": price})
# remove duplicates - can't add a player we already have
transfers_in, transfers_out = remove_duplicates(transfers_in, transfers_out)
# re-order both lists so they go DEF, FWD, GK, MID
transfers_in = sort_by_position(transfers_in)
transfers_out = sort_by_position(transfers_out)
transfer_list = [
{**transfers_in[i], **transfers_out[i]} for i in range(len(transfers_in))
]
return transfer_list
def calc_predicted_points_for_player(
player,
team_model,
df_player,
df_bonus,
df_saves,
df_cards,
season,
gw_range=None,
fixtures_behind=None,
tag="",
dbsession=session,
):
"""
Use the team-level model to get the probs of scoring or conceding
N goals, and player-level model to get the chance of player scoring
or assisting given that their team scores.
"""
if isinstance(player, int):
player = get_player(player, dbsession=dbsession)
message = "Points prediction for player {}".format(player)
if not gw_range:
# by default, go for next three matches
gw_range = list(range(NEXT_GAMEWEEK,
min(NEXT_GAMEWEEK + 3,
38))) # don't go beyond gw 38!
if fixtures_behind is None:
# default to getting recent minutes from the same number of matches we're
# predicting for
fixtures_behind = len(gw_range)
team = player.team(
season, gw_range[0]
) # assume player stays with same team from first gameweek in range
position = player.position(season)
fixtures = get_fixtures_for_player(player,
season,
gw_range=gw_range,
dbsession=dbsession)
# use same recent_minutes from previous gameweeks for all predictions
recent_minutes = get_recent_minutes_for_player(
player,
num_match_to_use=fixtures_behind,
season=season,
last_gw=min(gw_range) - 1,
dbsession=dbsession,
)
if len(recent_minutes) == 0:
# e.g. for gameweek 1
# this should now be dealt with in get_recent_minutes_for_player, so
# throw error if not.
# recent_minutes = estimate_minutes_from_prev_season(
# player, season=season, dbsession=session
# )
raise ValueError("Recent minutes is empty.")
expected_points = defaultdict(float) # default value is 0.
predictions = [] # list that will hold PlayerPrediction objects
for fixture in fixtures:
gameweek = fixture.gameweek
is_home = fixture.home_team == team
opponent = fixture.away_team if is_home else fixture.home_team
home_or_away = "at home" if is_home else "away"
message += "\ngameweek: {} vs {} {}".format(gameweek, opponent,
home_or_away)
points = 0.0
expected_points[gameweek] = points
if sum(recent_minutes) == 0:
# 'recent_minutes' contains the number of minutes that player played for
# in the past few matches. If these are all zero, we will for sure predict
# zero points for this player, so we don't need to call all the functions to
# calculate appearance points, defending points, attacking points.
points = 0.0
elif player.is_injured_or_suspended(season, gw_range[0], gameweek):
# Points for fixture will be zero if suspended or injured
points = 0.0
else:
# now loop over recent minutes and average
points = 0
for mins in recent_minutes:
points += (get_appearance_points(mins) + get_attacking_points(
player.player_id,
position,
team,
opponent,
is_home,
mins,
team_model,
df_player[position],
) + get_defending_points(position, team, opponent, is_home,
mins, team_model))
if df_bonus is not None:
points += get_bonus_points(player.player_id, mins,
df_bonus)
if df_cards is not None:
points += get_card_points(player.player_id, mins, df_cards)
if df_saves is not None:
points += get_save_points(position, player.player_id, mins,
df_saves)
points = points / len(recent_minutes)
# create the PlayerPrediction for this player+fixture
predictions.append(make_prediction(player, fixture, points, tag))
expected_points[gameweek] += points
# and return the per-gameweek predictions as a dict
message += "\nExpected points: {:.2f}".format(points)
print(message)
return predictions
def fill_playerscores_from_json(detail_data, season, dbsession=session):
for player_name in detail_data.keys():
# find the player id in the player table. If they're not
# there, then we don't care (probably not a current player).
player = get_player(player_name, dbsession=dbsession)
if not player:
print("Couldn't find player {}".format(player_name))
continue
print("SCORES {} {}".format(season, player))
# now loop through all the fixtures that player played in
for fixture_data in detail_data[player_name]:
# try to find the result in the result table
gameweek = int(fixture_data["gameweek"])
if "played_for" in fixture_data.keys():
played_for = fixture_data["played_for"]
else:
played_for = player.team(season, gameweek)
if not played_for:
continue
if fixture_data["was_home"] == "True":
was_home = True
elif fixture_data["was_home"] == "False":
was_home = False
else:
was_home = None
fixture = find_fixture(
played_for,
was_home=was_home,
other_team=fixture_data["opponent"],
gameweek=gameweek,
season=season,
kickoff_time=fixture_data["kickoff_time"],
dbsession=dbsession,
)
if not fixture or not fixture.result:
print(" Couldn't find result for {} in gw {}".format(
player, gameweek))
continue
ps = PlayerScore()
ps.player_team = played_for
ps.opponent = fixture_data["opponent"]
ps.goals = fixture_data["goals"]
ps.assists = fixture_data["assists"]
ps.bonus = fixture_data["bonus"]
ps.points = fixture_data["points"]
ps.conceded = fixture_data["conceded"]
ps.minutes = fixture_data["minutes"]
ps.player = player
ps.result = fixture.result
ps.fixture = fixture
# extended features
# get features excluding the core ones already populated above
extended_feats = [
col for col in ps.__table__.columns.keys() if col not in [
"id",
"player_team",
"opponent",
"goals",
"assists",
"bonus",
"points",
"conceded",
"minutes",
"player_id",
"result_id",
"fixture_id",
]
]
for feat in extended_feats:
try:
ps.__setattr__(feat, fixture_data[feat])
except KeyError:
pass
dbsession.add(ps)
dbsession.commit()
