def scrape_daily_games(game_id_list):
'''
this function scrapes the game ids provided in the list and returns a dict
with all the pbp and shift data for each game with game_id serving as the
key. It will also return a error_games list of the game where an error
occured in the scraping
Inputs:
game_id_list - a list of game ids to scrape
Outputs:
game_dict - dictionary containing shift and pbp data for each game
error_games - a list of game_ids where the scraper encountered errors
'''
#setup games dictionary to store the pbp of the games scraped from the day
#before
games_dict = {}
#create list to add games that errored out to write to file
error_games = []
for game in game_id_list:
scraped_data = hockey_scraper.scrape_games([game],
True,
data_format='Pandas')
#pull pbp, and shifts from the scraped data and write errors to the log
games_dict[str(game)] = {}
games_dict[str(game)]['pbp'] = scraped_data['pbp']
games_dict[str(game)]['shifts'] = scraped_data['shifts']
#if an error occurs from scraping the game, log the errors and save the game id
#to rescrape latter. Delete the key from the dictionary so the data is not
#mistakenly added to the database
if scraped_data['errors']:
error_games.append(game)
logging.info("All games scrapped")
logging.debug(games_dict.keys())
logging.info(f"Errors are {scraped_data['errors']}")
return games_dict, error_games
def main():
if (today.month >= 8):
currentseason = today.year
else:
currentseason = today.year - 1
scheduleUrl = str(
"https://statsapi.web.nhl.com/api/v1/schedule?teamId=15&startDate=" +
str(currentseason) + "-09-01&endDate=" + str(currentseason + 1) +
"-05-01")
print(scheduleUrl)
webUrl = urllib.request.urlopen(scheduleUrl)
print("result code: " + str(webUrl.getcode()))
if (webUrl.getcode() == 200):
data = webUrl.read()
gameJSON = getGame(data)
print(gameJSON)
else:
print("Received error, cannot parse results")
if gameJSON != None:
# gameUrl = urllib.request.urlopen(gameJSON[0])
# print ("result code: " + str(gameUrl.getcode()))
# if (gameUrl.getcode() == 200):
# gamedata = gameUrl.read()
# shotinfo = parseGame(gamedata)
# exportdict = constructJSON(shotinfo)
# with open('data.json', 'w+') as outfile:
# json.dump(exportdict, outfile, indent=4)
# else:
# print("Received error, cannot parse results")
scraped_data = hockey_scraper.scrape_games([gameJSON],
False,
data_format="Pandas")
print(scraped_data)
import hockey_scraper hockey_scraper.scrape_games(['2015020001', '2016020001'], True) hockey_scraper.scrape_seasons([2015, 2016], True)
def timeScrape(gameID, venue, gamefeed):
if venue == "home":
print("Ovechkin played at home")
elif venue == "away":
print("Ovechkin played on the road")
else:
print("venue error")
capsgoalies = []
oppsgoalies = []
# for i in gamefeed["gameData"]["players"].values():
# if i["primaryPosition"]["code"] == "G":
# if i["currentTeam"]["name"] == "Washington Capitals":
# capsgoalies.append(i["fullName"].upper())
# else:
# oppsgoalies.append(i["fullName"].upper())
for k, v in gamefeed["gameData"]["players"].items():
if venue == "home":
for i in gamefeed["liveData"]["boxscore"]["teams"][venue][
"goalies"]:
if k == "ID" + str(i):
capsgoalies.append(v["fullName"].upper())
for j in gamefeed["liveData"]["boxscore"]["teams"]["away"][
"goalies"]:
if k == "ID" + str(j):
oppsgoalies.append(v["fullName"].upper())
elif venue == "away":
for i in gamefeed["liveData"]["boxscore"]["teams"][venue][
"goalies"]:
if k == "ID" + str(i):
capsgoalies.append(v["fullName"].upper())
for j in gamefeed["liveData"]["boxscore"]["teams"]["home"][
"goalies"]:
if k == "ID" + str(j):
oppsgoalies.append(v["fullName"].upper())
print(capsgoalies, oppsgoalies)
scrape = hockey_scraper.scrape_games([gameID], True, data_format="Pandas")
print("Processing Game ", gameID)
pandashifts = scrape["shifts"]
pandaplays = scrape["pbp"]
# print(shifts)
pandashotlist = []
eventlist = ["GOAL", "SHOT", "MISS"]
shotdata = pandaplays.loc[pandaplays["Event"].isin(eventlist)]
roshotdata = shotdata.loc[shotdata["Period"] <= 4]
ovishotdata = roshotdata.loc[roshotdata["p1_ID"] == 8471214]
for i, j in ovishotdata.iterrows():
eventtime = parsePandaEventTime(j["Period"], j["Seconds_Elapsed"])
coordinates = convertCoord(gamefeed, venue, j["Period"], j["xC"],
j["yC"])
pandashotlist.append({
"type": j["Event"],
"time": eventtime,
"coordinates": coordinates
})
# print("Event", i, j["Event"], eventtime, "at", coordinates)
# jsonshotlist = []
# for i, j in enumerate(gamefeed["liveData"]["plays"]["allPlays"]):
# eventtime = parseJSONEventTime(j["about"]["period"],j["about"]["periodTime"])
# if "players" in j:
# if 1 <= j["about"]["period"] <= 4:
# if j["result"]["eventTypeId"] == "GOAL":
# if j["players"][0]["player"]["id"] == 8471214:
# coordinates = convertCoord(gamefeed, venue, j["about"]["period"], float(j["coordinates"]["x"]), float(j["coordinates"]["y"]))
# jsonshotlist.append({"type":j["result"]["eventTypeId"],"time":eventtime,"coordinates":coordinates})
# # print("Event", i, j["players"][0]["player"]["fullName"], j["result"]["event"], eventtime, "at", coordinates)
# elif j["result"]["eventTypeId"] == "SHOT":
# if j["players"][0]["player"]["id"] == 8471214:
# coordinates = convertCoord(gamefeed, venue, j["about"]["period"], float(j["coordinates"]["x"]), float(j["coordinates"]["y"]))
# jsonshotlist.append({"type":j["result"]["eventTypeId"],"time":eventtime,"coordinates":coordinates})
# # print("Event", i, j["players"][0]["player"]["fullName"], j["result"]["event"], eventtime, "at", coordinates)
# elif j["result"]["eventTypeId"] == "MISSED_SHOT":
# if j["players"][0]["player"]["id"] == 8471214:
# coordinates = convertCoord(gamefeed, venue, j["about"]["period"], float(j["coordinates"]["x"]), float(j["coordinates"]["y"]))
# jsonshotlist.append({"type":j["result"]["eventTypeId"],"time":eventtime,"coordinates":coordinates})
# # print("Event", i, j["players"][0]["player"]["fullName"], j["result"]["event"], eventtime, "at", coordinates)
# elif j["result"]["eventTypeId"] == "BLOCKED_SHOT":
# if j["players"][1]["player"]["id"] == 8471214:
# coordinates = convertCoord(gamefeed, venue, j["about"]["period"], float(j["coordinates"]["x"]), float(j["coordinates"]["y"]))
# jsonshotlist.append({"type":j["result"]["eventTypeId"],"time":eventtime,"coordinates":coordinates})
# # print("Event", i, j["players"][1]["player"]["fullName"], j["result"]["event"], eventtime, "at", coordinates)
# else:
# pass
# else:
# pass
# else:
# pass
shifts = []
ovishifts = []
for i, j in pandashifts.iterrows():
shift = parseShift(j["Period"], j["Start"], j["End"])
data = {}
data["player"] = j["Player"]
data["team"] = j["Team"]
data["start"] = shift[0]
data["end"] = shift[1]
shifts.append(data)
for i in shifts:
if i["player"] == "ALEX OVECHKIN":
ovishifts.append(i)
oppsOnIceWithOvi = []
capsOnIceWithOvi = []
for i, a in enumerate(ovishifts):
for b in shifts:
if b in capsOnIceWithOvi:
continue
elif b in oppsOnIceWithOvi:
continue
else:
if b["start"] < a["start"] and b["end"] > a["end"]:
if b["team"] == "WSH":
if b["start"] <= a["start"]:
start = a["start"]
else:
start = b["start"]
if b["end"] >= a["end"]:
end = a["end"]
else:
end = b["end"]
capsOnIceWithOvi.append({
"player": b["player"],
"team": b["team"],
"start": start,
"end": end
})
else:
if b["start"] <= a["start"]:
start = a["start"]
else:
start = b["start"]
if b["end"] >= a["end"]:
end = a["end"]
else:
end = b["end"]
oppsOnIceWithOvi.append({
"player": b["player"],
"team": b["team"],
"start": start,
"end": end
})
continue
if b["start"] >= a["start"] and b["start"] < a["end"]:
if b["team"] == "WSH":
if b["start"] <= a["start"]:
start = a["start"]
else:
start = b["start"]
if b["end"] >= a["end"]:
end = a["end"]
else:
end = b["end"]
capsOnIceWithOvi.append({
"player": b["player"],
"team": b["team"],
"start": start,
"end": end
})
else:
if b["start"] <= a["start"]:
start = a["start"]
else:
start = b["start"]
if b["end"] >= a["end"]:
end = a["end"]
else:
end = b["end"]
oppsOnIceWithOvi.append({
"player": b["player"],
"team": b["team"],
"start": start,
"end": end
})
continue
if b["end"] <= a["end"] and b["end"] > a["start"]:
if b["team"] == "WSH":
if b["start"] <= a["start"]:
start = a["start"]
else:
start = b["start"]
if b["end"] >= a["end"]:
end = a["end"]
else:
end = b["end"]
capsOnIceWithOvi.append({
"player": b["player"],
"team": b["team"],
"start": start,
"end": end
})
else:
if b["start"] <= a["start"]:
start = a["start"]
else:
start = b["start"]
if b["end"] >= a["end"]:
end = a["end"]
else:
end = b["end"]
oppsOnIceWithOvi.append({
"player": b["player"],
"team": b["team"],
"start": start,
"end": end
})
continue
strengthseconds = []
pandastrengthshotlist = []
for c in range(0, 3900):
# print(c,"seconds")
totCapsOnIce = 0
totOppsOnIce = 0
for d in capsOnIceWithOvi:
if d["player"] in capsgoalies:
pass
else:
if d["start"] <= c < d["end"]:
totCapsOnIce += 1
# print(d["player"])
for e in oppsOnIceWithOvi:
if e["player"] in oppsgoalies:
pass
else:
if e["start"] <= c < e["end"]:
totOppsOnIce += 1
# print(e["player"])
if totCapsOnIce > 6:
print(c, "seconds", totCapsOnIce, "Caps On Ice")
totCapsOnIce = 5
if totOppsOnIce > 6:
print(c, "seconds", totOppsOnIce, "Opponents On Ice")
totOppsOnIce = 5
strengthseconds.append(str(totCapsOnIce) + "v" + str(totOppsOnIce))
# print(str(totCapsOnIce) + "v" + str(totOppsOnIce))
for f in pandashotlist:
strength = strengthseconds[f["time"]]
if strength == "0v0":
strength = strengthseconds[f["time"] - 1]
event = {
"type": f["type"],
"time": f["time"],
"coordinates": f["coordinates"],
"strength": strength
}
pandastrengthshotlist.append(event)
strengths = {}
for item in strengthseconds:
if item in strengths.keys():
strengths[item] += 1
else:
strengths[item] = 1
return {"time": strengths, "events": pandastrengthshotlist}
def lambda_handler(event, context):
LAMBDA_GENERATOR = os.environ.get("LAMBDA_GENERATOR")
IS_SNS_TRIGGER = bool(event.get("Records"))
game_id_dict = get_game_id(event)
game_id_status = game_id_dict["status"]
if IS_SNS_TRIGGER:
msg = event['Records'][0]['Sns']['Message']
# Convert Message back to JSON Object
msg = json.loads(msg)
game_id = msg['gamePk']
period = msg['play']['about']['period']
goals = msg['play']['about']['goals']
home_score = goals['home']
away_score = goals['away']
else:
# Get scores directly from event payload
home_score = event.get("home_score")
away_score = event.get("away_score")
if not game_id_status:
logging.error(game_id_dict["msg"])
return {"status": False, "msg": game_id_dict["msg"]}
# If status is True, set the game_id variable
game_id = game_id_dict["game_id"]
# Check that the event period is greater than the last processed period
is_event_newer = is_event_period_newer(game_id, period)
if not is_event_newer:
logging.error(
"The event received for %s is not newer than the last "
"event recorded in the database - skip this record.", game_id)
return {
'status': 409,
'body': 'A shotmap was already produced for this event.'
}
# If all of the above checks pass, scrape the game.
scraped_data = hockey_scraper.scrape_games([game_id],
False,
data_format="Pandas")
pbp = scraped_data.get("pbp")
# fmt: off
cols_to_drop = [
'awayPlayer1', 'awayPlayer1_id', 'awayPlayer2', 'awayPlayer2_id',
'awayPlayer3', 'awayPlayer3_id', 'awayPlayer4', 'awayPlayer4_id',
'awayPlayer5', 'awayPlayer5_id', 'awayPlayer6', 'awayPlayer6_id',
'homePlayer1', 'homePlayer1_id', 'homePlayer2', 'homePlayer2_id',
'homePlayer3', 'homePlayer3_id', 'homePlayer4', 'homePlayer4_id',
'homePlayer5', 'homePlayer5_id', 'homePlayer6', 'homePlayer6_id',
'Description', 'Home_Coach', 'Away_Coach'
]
# fmt: on
pbp = pbp.drop(cols_to_drop, axis=1)
pbp.columns = map(str.lower, pbp.columns)
pbp_json = pbp.to_json()
payload = {
"pbp_json": pbp_json,
"game_id": game_id,
"testing": TESTING,
"home_score": home_score,
"away_score": away_score
}
small_payload = {
"game_id": game_id,
"testing": TESTING,
"home_score": home_score,
"away_score": away_score
}
logging.info(
"Scraping completed. Triggering the generator & twitter Lambda.")
invoke_response = lambda_client.invoke(FunctionName=LAMBDA_GENERATOR,
InvocationType="Event",
Payload=json.dumps(payload))
print(invoke_response)
return {'body': small_payload}
def dataScrape(gameID, venue, gamefeed):
if venue == "home":
print("Ovechkin played at home")
elif venue == "away":
print("Ovechkin played on the road")
else:
print("venue error")
shotsgoals = []
fenwickgoals = []
# corsigoals = []
shots = []
fenwick = []
# corsi = []
evshotsgoals = []
evfenwickgoals = []
# evcorsigoals = []
evshots = []
evfenwick = []
# evcorsi = []
ppshotsgoals = []
ppfenwickgoals = []
# ppcorsigoals = []
ppshots = []
ppfenwick = []
# ppcorsi = []
# print(gamefeed)
scrape = hockey_scraper.scrape_games([gameID], False, data_format="Pandas")
bigData = scrape["pbp"]
eventlist = ["GOAL", "SHOT", "MISS"]
shotdata = bigData.loc[bigData["Event"].isin(eventlist)]
roshotdata = shotdata.loc[shotdata["Period"] <= 4]
ovishotdata = roshotdata.loc[roshotdata["p1_ID"] == 8471214]
# print(ovishotdata)
for i, j in ovishotdata.iterrows():
coordinates = convertCoord(gamefeed, venue, j["Period"], j["xC"],
j["yC"])
# print(i, j["Description"], newcoord)
strength = j["Strength"]
strsplit = strength.split("x")
if venue == "home":
if int(strsplit[0]) >= 5 and int(strsplit[1]) >= 5:
if j["Event"] == "GOAL":
shotsgoals.append(1)
fenwickgoals.append(1)
# corsigoals.append(1)
shots.append(coordinates)
fenwick.append(coordinates)
# corsi.append(coordinates)
evshotsgoals.append(1)
evfenwickgoals.append(1)
# evcorsigoals.append(1)
evshots.append(coordinates)
evfenwick.append(coordinates)
# evcorsi.append(coordinates)
elif j["Event"] == "SHOT":
shotsgoals.append(0)
fenwickgoals.append(0)
# corsigoals.append(0)
shots.append(coordinates)
fenwick.append(coordinates)
# corsi.append(coordinates)
evshotsgoals.append(0)
evfenwickgoals.append(0)
# evcorsigoals.append(0)
evshots.append(coordinates)
evfenwick.append(coordinates)
# evcorsi.append(coordinates)
elif j["Event"] == "MISS":
fenwickgoals.append(0)
# corsigoals.append(0)
fenwick.append(coordinates)
# corsi.append(coordinates)
evfenwickgoals.append(0)
# evcorsigoals.append(0)
evfenwick.append(coordinates)
# evcorsi.append(coordinates)
elif int(strsplit[1]) == int(strsplit[0]):
if j["Event"] == "GOAL":
shotsgoals.append(1)
fenwickgoals.append(1)
# corsigoals.append(1)
shots.append(coordinates)
fenwick.append(coordinates)
# corsi.append(coordinates)
evshotsgoals.append(1)
evfenwickgoals.append(1)
# evcorsigoals.append(1)
evshots.append(coordinates)
evfenwick.append(coordinates)
# evcorsi.append(coordinates)
elif j["Event"] == "SHOT":
shotsgoals.append(0)
fenwickgoals.append(0)
# corsigoals.append(0)
shots.append(coordinates)
fenwick.append(coordinates)
# corsi.append(coordinates)
evshotsgoals.append(0)
evfenwickgoals.append(0)
# evcorsigoals.append(0)
evshots.append(coordinates)
evfenwick.append(coordinates)
# evcorsi.append(coordinates)
elif j["Event"] == "MISS":
fenwickgoals.append(0)
# corsigoals.append(0)
fenwick.append(coordinates)
# corsi.append(coordinates)
evfenwickgoals.append(0)
# evcorsigoals.append(0)
evfenwick.append(coordinates)
# evcorsi.append(coordinates)
elif int(strsplit[1]) < int(strsplit[0]):
if j["Event"] == "GOAL":
shotsgoals.append(1)
fenwickgoals.append(1)
# corsigoals.append(1)
shots.append(coordinates)
fenwick.append(coordinates)
# corsi.append(coordinates)
ppshotsgoals.append(1)
ppfenwickgoals.append(1)
# ppcorsigoals.append(1)
ppshots.append(coordinates)
ppfenwick.append(coordinates)
# ppcorsi.append(coordinates)
elif j["Event"] == "SHOT":
shotsgoals.append(0)
fenwickgoals.append(0)
# corsigoals.append(0)
shots.append(coordinates)
fenwick.append(coordinates)
# corsi.append(coordinates)
ppshotsgoals.append(0)
ppfenwickgoals.append(0)
# ppcorsigoals.append(0)
ppshots.append(coordinates)
ppfenwick.append(coordinates)
# ppcorsi.append(coordinates)
elif j["Event"] == "MISS":
fenwickgoals.append(0)
# corsigoals.append(0)
fenwick.append(coordinates)
# corsi.append(coordinates)
ppfenwickgoals.append(0)
# ppcorsigoals.append(0)
ppfenwick.append(coordinates)
# ppcorsi.append(coordinates)
else:
pass
elif venue == "away":
if int(strsplit[0]) >= 5 and int(strsplit[1]) >= 5:
if j["Event"] == "GOAL":
shotsgoals.append(1)
fenwickgoals.append(1)
# corsigoals.append(1)
shots.append(coordinates)
fenwick.append(coordinates)
# corsi.append(coordinates)
evshotsgoals.append(1)
evfenwickgoals.append(1)
# evcorsigoals.append(1)
evshots.append(coordinates)
evfenwick.append(coordinates)
# evcorsi.append(coordinates)
elif j["Event"] == "SHOT":
shotsgoals.append(0)
fenwickgoals.append(0)
# corsigoals.append(0)
shots.append(coordinates)
fenwick.append(coordinates)
# corsi.append(coordinates)
evshotsgoals.append(0)
evfenwickgoals.append(0)
# evcorsigoals.append(0)
evshots.append(coordinates)
evfenwick.append(coordinates)
# evcorsi.append(coordinates)
elif j["Event"] == "MISS":
fenwickgoals.append(0)
# corsigoals.append(0)
fenwick.append(coordinates)
# corsi.append(coordinates)
evfenwickgoals.append(0)
# evcorsigoals.append(0)
evfenwick.append(coordinates)
# evcorsi.append(coordinates)
elif int(strsplit[0]) == int(strsplit[1]):
if j["Event"] == "GOAL":
shotsgoals.append(1)
fenwickgoals.append(1)
# corsigoals.append(1)
shots.append(coordinates)
fenwick.append(coordinates)
# corsi.append(coordinates)
evshotsgoals.append(1)
evfenwickgoals.append(1)
# evcorsigoals.append(1)
evshots.append(coordinates)
evfenwick.append(coordinates)
# evcorsi.append(coordinates)
elif j["Event"] == "SHOT":
shotsgoals.append(0)
fenwickgoals.append(0)
# corsigoals.append(0)
shots.append(coordinates)
fenwick.append(coordinates)
# corsi.append(coordinates)
evshotsgoals.append(0)
evfenwickgoals.append(0)
# evcorsigoals.append(0)
evshots.append(coordinates)
evfenwick.append(coordinates)
# evcorsi.append(coordinates)
elif j["Event"] == "MISS":
fenwickgoals.append(0)
# corsigoals.append(0)
fenwick.append(coordinates)
# corsi.append(coordinates)
evfenwickgoals.append(0)
# evcorsigoals.append(0)
evfenwick.append(coordinates)
# evcorsi.append(coordinates)
elif int(strsplit[0]) < int(strsplit[1]):
if j["Event"] == "GOAL":
shotsgoals.append(1)
fenwickgoals.append(1)
# corsigoals.append(1)
shots.append(coordinates)
fenwick.append(coordinates)
# corsi.append(coordinates)
ppshotsgoals.append(1)
ppfenwickgoals.append(1)
# ppcorsigoals.append(1)
ppshots.append(coordinates)
ppfenwick.append(coordinates)
# ppcorsi.append(coordinates)
elif j["Event"] == "SHOT":
shotsgoals.append(0)
fenwickgoals.append(0)
# corsigoals.append(0)
shots.append(coordinates)
fenwick.append(coordinates)
# corsi.append(coordinates)
ppshotsgoals.append(0)
ppfenwickgoals.append(0)
# ppcorsigoals.append(0)
ppshots.append(coordinates)
ppfenwick.append(coordinates)
# ppcorsi.append(coordinates)
elif j["Event"] == "MISS":
fenwickgoals.append(0)
# corsigoals.append(0)
fenwick.append(coordinates)
# corsi.append(coordinates)
ppfenwickgoals.append(0)
# ppcorsigoals.append(0)
ppfenwick.append(coordinates)
# ppcorsi.append(coordinates)
else:
pass
avgx = 0
totalx = 0
for i in range(0, len(fenwick)):
totalx = totalx + fenwick[i][0]
if len(fenwick) > 0:
avgx = totalx / len(fenwick)
if avgx > 100:
# print("-------------------- INVERT HERE --------------------")
for fen in range(0, len(fenwick)):
fenwick[fen][0] = -fenwick[fen][0] + 178
fenwick[fen][1] = -fenwick[fen][1]
print(shots)
print(fenwick)
print(evshots)
print(evfenwick)
print(ppshots)
print(ppfenwick)
# print("---------------- INVERSION COMPLETE ----------------")
return [
shotsgoals, fenwickgoals, shots, fenwick, evshotsgoals, evfenwickgoals,
evshots, evfenwick, ppshotsgoals, ppfenwickgoals, ppshots, ppfenwick
]
import hockey_scraper hockey_scraper.scrape_games([2015020031], True)