def try_live():
for game in pbpmodels.Game.objects.filter(gameState__in=[5, 6, 7]):
pbps = pbpmodels.PlayByPlay.objects.filter(gamePk=game)
eventIds = {}
for pbp in pbps:
if pbp.eventId not in eventIds:
eventIds[pbp.eventId] = pbp
url = LIVE_BASE + str(game.season) + "/" + str(game.gamePk) + "/PlayByPlay.json"
j = json.loads(api_calls.get_url(url))
count = set()
if "data" in j:
data = j["data"]
if "game" in data:
jgame = data["game"]
if "plays" in jgame:
for play in jgame["plays"]["play"]:
if play["eventid"] in eventIds:
pbp = eventIds[play["eventid"]]
count.add(play["eventid"])
missing = set()
for ei in eventIds:
if ei not in count:
missing.add(eventIds[ei].playType)
print missing
break
def try_live():
for game in pbpmodels.Game.objects.filter(gameState__in=[5, 6, 7]):
pbps = pbpmodels.PlayByPlay.objects.filter(gamePk=game)
eventIds = {}
for pbp in pbps:
if pbp.eventId not in eventIds:
eventIds[pbp.eventId] = pbp
url = LIVE_BASE + str(game.season) + "/" + str(
game.gamePk) + "/PlayByPlay.json"
j = json.loads(api_calls.get_url(url))
count = set()
if "data" in j:
data = j["data"]
if "game" in data:
jgame = data["game"]
if "plays" in jgame:
for play in jgame["plays"]["play"]:
if play["eventid"] in eventIds:
pbp = eventIds[play["eventid"]]
count.add(play["eventid"])
missing = set()
for ei in eventIds:
if ei not in count:
missing.add(eventIds[ei].playType)
print missing
break
def get_old():
for game in pbpmodels.Game.objects.filter(gameState__in=[5, 6, 7]):
eventIdxs = {}
pbps = pbpmodels.PlayByPlay.objects.values("eventIdx", "id").filter(gamePk=game)
for pbp in pbps:
if pbp["eventIdx"] not in eventIdxs:
eventIdxs[pbp["eventIdx"]] = pbp["id"]
print game.gamePk
if not pbpmodels.PlayerOnIce.objects.filter(play_id__in=eventIdxs.values()).exists():
playerstats = pbpmodels.PlayerGameStats.objects.values("team_id", "player__fullName", "player_id").filter(game=game)
hp = {}
ap = {}
for ps in playerstats:
if ps["team_id"] == game.homeTeam_id:
hp[ps["player__fullName"].upper()] = ps["player_id"]
else:
ap[ps["player__fullName"].upper()] = ps["player_id"]
goaliestats = pbpmodels.GoalieGameStats.objects.values("team_id", "player__fullName", "player_id").filter(game=game)
for gs in goaliestats:
if gs["team_id"] == game.homeTeam_id:
hp[gs["player__fullName"].upper()] = gs["player_id"]
else:
ap[gs["player__fullName"].upper()] = gs["player_id"]
url = BASE + str(game.season) + "/PL0" + str(game.gamePk)[5:] + ".HTM"
data = api_calls.get_url(url)
soup = BeautifulSoup(data, 'html.parser')
#table = soup.find('table', attrs={'id': 'GameInfo'})
evens = soup.find_all('tr', attrs={'class': 'evenColor'})
count = 0
saved = []
with transaction.atomic():
for row in evens:
cols = row.find_all('td', recursive=False)
fonts = row.find_all('font')
fcols = [ele.text.strip().replace("\n", "") for ele in cols]
eventIdx = int(fcols[0])
if eventIdx in eventIdxs:
players = fcols[6:]
away = players[0]
home = players[1]
away = [x[0:-1] for x in away.replace(u'\xa0', " ").split(" ")]
home = [x[0:-1] for x in home.replace(u'\xa0', " ").split(" ")]
awayNames = {}
homeNames = {}
for f in fonts:
title = f["title"].split(" - ")[-1]
number = f.text
if number in away and number not in awayNames:
awayNames[number] = title
else:
homeNames[number] = title
acount = 1
players = set()
for anum in away:
if len(anum) > 0:
pbpdict = {}
pbpdict["play_id"] = eventIdxs[eventIdx]
pbpdict["game_id"] = game.gamePk
anum = int(anum)
player = getPlayer(ap, awayNames, anum) #ap[awayNames[str(anum)]]
if player not in players:
players.add(player)
pbpdict["player_id"] = player
acount += 1
try:
pbpp = pbpmodels.PlayerOnIce(**pbpdict)
#pbpp.save()
saved.append(pbpp)
except TypeError:
pass
hcount = 1
for hnum in home:
if len(hnum) > 0:
pbpdict = {}
pbpdict["play_id"] = eventIdxs[eventIdx]
pbpdict["game_id"] = game.gamePk
hnum = int(hnum)
player = getPlayer(hp, homeNames, hnum)
if player not in players:
players.add(player)
pbpdict["player_id"] = player
hcount += 1
try:
pbpp = pbpmodels.PlayerOnIce(**pbpdict)
#pbpp.save()
saved.append(pbpp)
except TypeError:
pass
pbpmodels.PlayerOnIce.objects.bulk_create(saved)
def update_game(game, players):
missing_players = set()
allpgss = []
allperiods = []
allplaybyplay = []
allplayers = []
homeMissed = 0
awayMissed = 0
# Get live game data
j = json.loads(api_calls.get_game(game.gamePk))
gd = j["gameData"]
ld = j["liveData"]
boxScore = ld["boxscore"]
lineScore = ld["linescore"]
# Update gameData
game.dateTime = gd["datetime"]["dateTime"]
if "endDateTime" in gd["datetime"]:
game.endDateTime = gd["datetime"]["endDateTime"]
if game.gameState == 1 and gd["status"]["codedGameState"] != 1:
new_game = True
else:
new_game = False
game.gameState = gd["status"]["codedGameState"]
# Get linescore information
game.homeScore = lineScore["teams"]["home"]["goals"]
game.awayScore = lineScore["teams"]["away"]["goals"]
game.homeShots = lineScore["teams"]["home"]["shotsOnGoal"]
game.awayShots = lineScore["teams"]["away"]["shotsOnGoal"]
# Get boxscore information
if 'teamSkaterStats' not in boxScore['teams']['home']['teamStats']:
return False
home = boxScore["teams"]["home"]["teamStats"]["teamSkaterStats"]
away = boxScore["teams"]["away"]["teamStats"]["teamSkaterStats"]
game.homePIM = home["pim"]
game.awayPIM = away["pim"]
game.homePPGoals = home["powerPlayGoals"]
game.awayPPGoals = away["powerPlayGoals"]
game.homePPOpportunities = home["powerPlayOpportunities"]
game.awayPPOpportunities = away["powerPlayOpportunities"]
game.homeFaceoffPercentage = home["faceOffWinPercentage"]
game.awayFaceoffPercentage = away["faceOffWinPercentage"]
game.homeBlocked = home["blocked"]
game.awayBlocked = away["blocked"]
game.homeTakeaways = home["takeaways"]
game.awayTakeaways = away["takeaways"]
game.homeGiveaways = home["giveaways"]
game.awayGiveaways = away["giveaways"]
game.homeHits = home["hits"]
game.awayHits = away["hits"]
cperiod = 1
# Delete old data
if pbpmodels.GamePeriod.objects.filter(game=game).count() > 0:
pbpmodels.GamePeriod.objects.filter(game=game).delete()
if pbpmodels.PlayerInPlay.objects.filter(game=game).count() > 0:
pbpmodels.PlayerInPlay.objects.filter(game=game).delete()
if pbpmodels.PlayByPlay.objects.filter(gamePk=game).count() > 0:
pbpmodels.PlayByPlay.objects.filter(gamePk=game).delete()
if pbpmodels.PlayerGameStats.objects.filter(game=game).count() > 0:
pbpmodels.PlayerGameStats.objects.filter(game=game).delete()
for period in lineScore["periods"]:
p = pbpmodels.GamePeriod()
p.game = game
p.period = period["num"]
if period["num"] > cperiod:
cperiod = period["num"]
if "startTime" in period:
p.startTime = period["startTime"]
if "endTime" in period:
p.endTime = period["endTime"]
p.homeScore = period["home"]["goals"]
p.homeShots = period["home"]["shotsOnGoal"]
p.awayScore = period["away"]["goals"]
p.awayShots = period["away"]["shotsOnGoal"]
allperiods.append(p)
if lineScore["hasShootout"]:
sinfo = lineScore["shootoutInfo"]
try:
s = pbpmodels.Shootout.objects.get(game=game)
except:
s = pbpmodels.Shootout()
s.game = game
s.awayScores = sinfo["away"]["scores"]
s.awayAttempts = sinfo["away"]["attempts"]
s.homeScores = sinfo["home"]["scores"]
s.homeAttempts = sinfo["home"]["attempts"]
s.save()
homeSkaters = j["liveData"]["boxscore"]["teams"]["home"]["skaters"]
homeGoalies = j["liveData"]["boxscore"]["teams"]["home"]["goalies"]
homeOnIce = j["liveData"]["boxscore"]["teams"]["home"]["onIce"]
homeScratches = j["liveData"]["boxscore"]["teams"]["home"]["scratches"]
awaySkaters = j["liveData"]["boxscore"]["teams"]["away"]["skaters"]
awayGoalies = j["liveData"]["boxscore"]["teams"]["away"]["goalies"]
awayOnIce = j["liveData"]["boxscore"]["teams"]["away"]["onIce"]
awayScratches = j["liveData"]["boxscore"]["teams"]["away"]["scratches"]
homeIds = set(homeSkaters + homeGoalies + homeOnIce + homeScratches)
awayIds = set(awaySkaters + awayGoalies + awayOnIce + awayScratches)
gd = j["gameData"]
# Player Info
pinfo = gd["players"]
for sid in pinfo: # I swear that's not a Crosby reference
iid = int(sid.replace("ID", ""))
if iid not in players:
if iid in homeIds:
team = game.homeTeam
elif iid in awayIds:
team = game.awayTeam
else:
raise Exception('iid {} not in players'.format(iid))
player = ingest_player(pinfo[sid], team.id)
players[player.id] = player
# liveData
ld = j["liveData"]
lineScore = ld["linescore"]
# Plays
playid = pbpmodels.PlayByPlay.objects.values('id').latest('id')['id'] + 1
homeScore = 0
awayScore = 0
for play in ld["plays"]["allPlays"]:
about = play["about"]
if "players" in play:
pplayers = play["players"]
else:
pplayers = {}
result = play["result"]
coordinates = play["coordinates"]
p = pbpmodels.PlayByPlay()
p.id = playid
p.gamePk = game
p.eventId = about["eventId"]
p.eventIdx = about["eventIdx"]
p.period = about["period"]
p.periodTime = about["periodTime"]
p.dateTime = about["dateTime"]
p.playType = result["eventTypeId"]
p.playDescription = result["description"]
if "team" in play:
p.team_id = play["team"]["id"]
if result["eventTypeId"] == "MISSED_SHOT":
if play["team"]["id"] == game.homeTeam_id:
homeMissed += 1
else:
awayMissed += 1
if "secondaryType" in result:
if p.playType == "PENALTY":
p.penaltyType = result["secondaryType"]
else:
p.shotType = result["secondaryType"]
if p.playType == "PENALTY":
p.penaltySeverity = result["penaltySeverity"]
p.penaltyMinutes = result["penaltyMinutes"]
if "strength" in result:
p.strength = result["strength"]["code"]
if "x" in coordinates and "y" in coordinates:
p.xcoord = coordinates["x"]
p.ycoord = coordinates["y"]
if result["eventTypeId"] == "GOAL":
if play["team"]["id"] == game.homeTeam_id:
homeScore += 1
else:
awayScore += 1
p.homeScore = homeScore
p.awayScore = awayScore
allplaybyplay.append(p)
assist_found = False
for pp in pplayers:
skip_play = False # For all those preseason players that don't exist!
poi = pbpmodels.PlayerInPlay()
poi.play_id = playid
poi.game = game
try:
playerfound = pmodels.Player.objects.get(id=pp["player"]["id"])
except:
try:
if pp["player"]["id"] not in missing_players:
print "API CALL"
playerdata = json.loads(
api_calls.get_player(pp["player"]["id"]))
if len(playerdata.keys()) > 0:
playerfound = ingest_player(playerdata)
else:
skip_play = True
missing_players.add(pp["player"]["id"])
else:
skip_play = True
except:
skip_play = True
missing_players.add(pp["player"]["id"])
poi.player_id = pp["player"]["id"]
if assist_found is True and fancystats.player.get_player_type(
pp["playerType"]) == 6:
poi.player_type = 16
poi.eventId = about["eventId"]
poi.game_id = game.gamePk
else:
poi.player_type = fancystats.player.get_player_type(
pp["playerType"])
if poi.player_type == 6:
assist_found = True
if skip_play is False:
allplayers.append(poi)
playid += 1
game.homeMissed = homeMissed
game.awayMissed = awayMissed
game.save()
pbpmodels.GamePeriod.objects.bulk_create(allperiods)
pbpmodels.PlayByPlay.objects.bulk_create(allplaybyplay)
pbpmodels.PlayerInPlay.objects.bulk_create(allplayers)
hp = boxScore["teams"]["home"]["players"]
ap = boxScore["teams"]["away"]["players"]
homegoalies = ld["boxscore"]["teams"]["home"]["players"]
awaygoalies = ld["boxscore"]["teams"]["away"]["players"]
hometeam = ld["boxscore"]["teams"]["home"]["team"]["id"]
awayteam = ld["boxscore"]["teams"]["away"]["team"]["id"]
goaliestats = []
goaliestats.extend(
checkGoalies(homegoalies, game.gamePk, hometeam, cperiod))
goaliestats.extend(
checkGoalies(awaygoalies, game.gamePk, awayteam, cperiod))
allpgss.extend(
set_player_stats(hp, game.homeTeam, game, players, cperiod, new_game))
allpgss.extend(
set_player_stats(ap, game.awayTeam, game, players, cperiod, new_game))
pbpmodels.PlayerGameStats.objects.bulk_create(allpgss)
# Find any existing POI data and delete
pbpmodels.PlayerOnIce.objects.filter(game=game).delete()
# Get player on ice data
eventIdxs = {}
for pbp in allplaybyplay:
periodTime = str(pbp.periodTime)
if pbp.period not in eventIdxs:
eventIdxs[pbp.period] = {}
if periodTime not in eventIdxs[pbp.period]:
eventIdxs[pbp.period][periodTime] = {}
if fancystats.constants.events[pbp.playType] not in eventIdxs[
pbp.period][periodTime]:
eventIdxs[pbp.period][periodTime][fancystats.constants.events[
pbp.playType]] = [
pbp.id,
]
else:
eventIdxs[pbp.period][periodTime][fancystats.constants.events[
pbp.playType]].append(pbp.id)
hp = {}
ap = {}
for ps in allpgss:
if ps.team_id == game.homeTeam_id:
hp[ps.player.fullName.upper()] = ps.player_id
else:
ap[ps.player.fullName.upper()] = ps.player_id
for gs in goaliestats:
if gs.team_id == game.homeTeam_id:
hp[gs.player.fullName.upper()] = gs.player_id
else:
ap[gs.player.fullName.upper()] = gs.player_id
url = BASE + str(game.season) + "/PL0" + str(game.gamePk)[5:] + ".HTM"
data = api_calls.get_url(url)
soup = BeautifulSoup(data, 'html.parser')
evens = soup.find_all('tr', attrs={'class': 'evenColor'})
count = 0
saved = []
for row in evens:
backup_names = {}
cols = row.find_all('td', recursive=False)
fonts = row.find_all('font')
if len(list(cols[3].strings)) >= 1:
time = list(cols[3].strings)[0]
if len(time) < 5:
time = "0" + time
for ele in fonts:
if ele.has_attr("title"):
title = ele.attrs["title"].split(" - ")[1]
number = ele.text
if number in backup_names:
backup_names[number + "H"] = title
else:
backup_names[number] = title
fcols = [ele.text.strip().replace("\n", "") for ele in cols]
eventIdx = int(fcols[1])
playType = fcols[4]
if eventIdx in eventIdxs and time in eventIdxs[
eventIdx] and playType in eventIdxs[eventIdx][time]:
players = fcols[6:]
away = players[0]
home = players[1]
away = [x[0:-1] for x in away.replace(u'\xa0', " ").split(" ")]
home = [x[0:-1] for x in home.replace(u'\xa0', " ").split(" ")]
awayNames = {}
homeNames = {}
for f in fonts:
if "title" in f:
title = f["title"].split(" - ")[-1]
number = f.text
if number in away and number not in awayNames:
awayNames[number] = title
else:
homeNames[number] = title
acount = 1
players = set()
for anum in away:
if len(anum) > 0:
for play_id in eventIdxs[eventIdx][time][playType]:
pbpdict = {}
pbpdict["play_id"] = play_id
pbpdict["game_id"] = game.gamePk
anum = int(anum)
try:
player = getPlayer(
ap, awayNames, anum, backup_names,
True) #ap[awayNames[str(anum)]]
if player not in players:
players.add(player)
pbpdict["player_id"] = player
acount += 1
pbpp = pbpmodels.PlayerOnIce(**pbpdict)
#pbpp.save()
saved.append(pbpp)
except:
pass
hcount = 1
for hnum in home:
if len(hnum) > 0:
for play_id in eventIdxs[eventIdx][time][playType]:
pbpdict = {}
pbpdict["play_id"] = play_id
pbpdict["game_id"] = game.gamePk
# fix yo formatting nhl dot com
hnum = int(
str(hnum).replace("=\"center\">",
"").replace("C", ""))
try:
player = getPlayer(hp, homeNames, hnum,
backup_names, False)
if player not in players:
players.add(player)
pbpdict["player_id"] = player
hcount += 1
pbpp = pbpmodels.PlayerOnIce(**pbpdict)
#pbpp.save()
saved.append(pbpp)
except:
pass
# Remove so there are no duplicates, first entry will have the most data
eventIdxs[eventIdx][time].pop(playType, None)
pbpmodels.PlayerOnIce.objects.bulk_create(saved)
if game.gameState in [6, 7, 8, "6", "7", "8"]:
return True
return False
def get_old():
for game in pbpmodels.Game.objects.filter(gameState__in=[5, 6, 7]):
eventIdxs = {}
pbps = pbpmodels.PlayByPlay.objects.values("eventIdx",
"id").filter(gamePk=game)
for pbp in pbps:
if pbp["eventIdx"] not in eventIdxs:
eventIdxs[pbp["eventIdx"]] = pbp["id"]
print game.gamePk
if not pbpmodels.PlayerOnIce.objects.filter(
play_id__in=eventIdxs.values()).exists():
playerstats = pbpmodels.PlayerGameStats.objects.values(
"team_id", "player__fullName", "player_id").filter(game=game)
hp = {}
ap = {}
for ps in playerstats:
if ps["team_id"] == game.homeTeam_id:
hp[ps["player__fullName"].upper()] = ps["player_id"]
else:
ap[ps["player__fullName"].upper()] = ps["player_id"]
goaliestats = pbpmodels.GoalieGameStats.objects.values(
"team_id", "player__fullName", "player_id").filter(game=game)
for gs in goaliestats:
if gs["team_id"] == game.homeTeam_id:
hp[gs["player__fullName"].upper()] = gs["player_id"]
else:
ap[gs["player__fullName"].upper()] = gs["player_id"]
url = BASE + str(game.season) + "/PL0" + str(
game.gamePk)[5:] + ".HTM"
data = api_calls.get_url(url)
soup = BeautifulSoup(data, 'html.parser')
#table = soup.find('table', attrs={'id': 'GameInfo'})
evens = soup.find_all('tr', attrs={'class': 'evenColor'})
count = 0
saved = []
with transaction.atomic():
for row in evens:
cols = row.find_all('td', recursive=False)
fonts = row.find_all('font')
fcols = [
ele.text.strip().replace("\n", "") for ele in cols
]
eventIdx = int(fcols[0])
if eventIdx in eventIdxs:
players = fcols[6:]
away = players[0]
home = players[1]
away = [
x[0:-1]
for x in away.replace(u'\xa0', " ").split(" ")
]
home = [
x[0:-1]
for x in home.replace(u'\xa0', " ").split(" ")
]
awayNames = {}
homeNames = {}
for f in fonts:
title = f["title"].split(" - ")[-1]
number = f.text
if number in away and number not in awayNames:
awayNames[number] = title
else:
homeNames[number] = title
acount = 1
players = set()
for anum in away:
if len(anum) > 0:
pbpdict = {}
pbpdict["play_id"] = eventIdxs[eventIdx]
pbpdict["game_id"] = game.gamePk
anum = int(anum)
player = getPlayer(
ap, awayNames,
anum) #ap[awayNames[str(anum)]]
if player not in players:
players.add(player)
pbpdict["player_id"] = player
acount += 1
try:
pbpp = pbpmodels.PlayerOnIce(**pbpdict)
#pbpp.save()
saved.append(pbpp)
except TypeError:
pass
hcount = 1
for hnum in home:
if len(hnum) > 0:
pbpdict = {}
pbpdict["play_id"] = eventIdxs[eventIdx]
pbpdict["game_id"] = game.gamePk
hnum = int(hnum)
player = getPlayer(hp, homeNames, hnum)
if player not in players:
players.add(player)
pbpdict["player_id"] = player
hcount += 1
try:
pbpp = pbpmodels.PlayerOnIce(**pbpdict)
#pbpp.save()
saved.append(pbpp)
except TypeError:
pass
pbpmodels.PlayerOnIce.objects.bulk_create(saved)