def scrape_daily_games(game_id_list): ''' this function scrapes the game ids provided in the list and returns a dict with all the pbp and shift data for each game with game_id serving as the key. It will also return a error_games list of the game where an error occured in the scraping Inputs: game_id_list - a list of game ids to scrape Outputs: game_dict - dictionary containing shift and pbp data for each game error_games - a list of game_ids where the scraper encountered errors ''' #setup games dictionary to store the pbp of the games scraped from the day #before games_dict = {} #create list to add games that errored out to write to file error_games = [] for game in game_id_list: scraped_data = hockey_scraper.scrape_games([game], True, data_format='Pandas') #pull pbp, and shifts from the scraped data and write errors to the log games_dict[str(game)] = {} games_dict[str(game)]['pbp'] = scraped_data['pbp'] games_dict[str(game)]['shifts'] = scraped_data['shifts'] #if an error occurs from scraping the game, log the errors and save the game id #to rescrape latter. Delete the key from the dictionary so the data is not #mistakenly added to the database if scraped_data['errors']: error_games.append(game) logging.info("All games scrapped") logging.debug(games_dict.keys()) logging.info(f"Errors are {scraped_data['errors']}") return games_dict, error_games
def main(): if (today.month >= 8): currentseason = today.year else: currentseason = today.year - 1 scheduleUrl = str( "https://statsapi.web.nhl.com/api/v1/schedule?teamId=15&startDate=" + str(currentseason) + "-09-01&endDate=" + str(currentseason + 1) + "-05-01") print(scheduleUrl) webUrl = urllib.request.urlopen(scheduleUrl) print("result code: " + str(webUrl.getcode())) if (webUrl.getcode() == 200): data = webUrl.read() gameJSON = getGame(data) print(gameJSON) else: print("Received error, cannot parse results") if gameJSON != None: # gameUrl = urllib.request.urlopen(gameJSON[0]) # print ("result code: " + str(gameUrl.getcode())) # if (gameUrl.getcode() == 200): # gamedata = gameUrl.read() # shotinfo = parseGame(gamedata) # exportdict = constructJSON(shotinfo) # with open('data.json', 'w+') as outfile: # json.dump(exportdict, outfile, indent=4) # else: # print("Received error, cannot parse results") scraped_data = hockey_scraper.scrape_games([gameJSON], False, data_format="Pandas") print(scraped_data)
import hockey_scraper hockey_scraper.scrape_games(['2015020001', '2016020001'], True) hockey_scraper.scrape_seasons([2015, 2016], True)
def timeScrape(gameID, venue, gamefeed): if venue == "home": print("Ovechkin played at home") elif venue == "away": print("Ovechkin played on the road") else: print("venue error") capsgoalies = [] oppsgoalies = [] # for i in gamefeed["gameData"]["players"].values(): # if i["primaryPosition"]["code"] == "G": # if i["currentTeam"]["name"] == "Washington Capitals": # capsgoalies.append(i["fullName"].upper()) # else: # oppsgoalies.append(i["fullName"].upper()) for k, v in gamefeed["gameData"]["players"].items(): if venue == "home": for i in gamefeed["liveData"]["boxscore"]["teams"][venue][ "goalies"]: if k == "ID" + str(i): capsgoalies.append(v["fullName"].upper()) for j in gamefeed["liveData"]["boxscore"]["teams"]["away"][ "goalies"]: if k == "ID" + str(j): oppsgoalies.append(v["fullName"].upper()) elif venue == "away": for i in gamefeed["liveData"]["boxscore"]["teams"][venue][ "goalies"]: if k == "ID" + str(i): capsgoalies.append(v["fullName"].upper()) for j in gamefeed["liveData"]["boxscore"]["teams"]["home"][ "goalies"]: if k == "ID" + str(j): oppsgoalies.append(v["fullName"].upper()) print(capsgoalies, oppsgoalies) scrape = hockey_scraper.scrape_games([gameID], True, data_format="Pandas") print("Processing Game ", gameID) pandashifts = scrape["shifts"] pandaplays = scrape["pbp"] # print(shifts) pandashotlist = [] eventlist = ["GOAL", "SHOT", "MISS"] shotdata = pandaplays.loc[pandaplays["Event"].isin(eventlist)] roshotdata = shotdata.loc[shotdata["Period"] <= 4] ovishotdata = roshotdata.loc[roshotdata["p1_ID"] == 8471214] for i, j in ovishotdata.iterrows(): eventtime = parsePandaEventTime(j["Period"], j["Seconds_Elapsed"]) coordinates = convertCoord(gamefeed, venue, j["Period"], j["xC"], j["yC"]) pandashotlist.append({ "type": j["Event"], "time": eventtime, "coordinates": coordinates }) # print("Event", i, j["Event"], eventtime, "at", coordinates) # jsonshotlist = [] # for i, j in enumerate(gamefeed["liveData"]["plays"]["allPlays"]): # eventtime = parseJSONEventTime(j["about"]["period"],j["about"]["periodTime"]) # if "players" in j: # if 1 <= j["about"]["period"] <= 4: # if j["result"]["eventTypeId"] == "GOAL": # if j["players"][0]["player"]["id"] == 8471214: # coordinates = convertCoord(gamefeed, venue, j["about"]["period"], float(j["coordinates"]["x"]), float(j["coordinates"]["y"])) # jsonshotlist.append({"type":j["result"]["eventTypeId"],"time":eventtime,"coordinates":coordinates}) # # print("Event", i, j["players"][0]["player"]["fullName"], j["result"]["event"], eventtime, "at", coordinates) # elif j["result"]["eventTypeId"] == "SHOT": # if j["players"][0]["player"]["id"] == 8471214: # coordinates = convertCoord(gamefeed, venue, j["about"]["period"], float(j["coordinates"]["x"]), float(j["coordinates"]["y"])) # jsonshotlist.append({"type":j["result"]["eventTypeId"],"time":eventtime,"coordinates":coordinates}) # # print("Event", i, j["players"][0]["player"]["fullName"], j["result"]["event"], eventtime, "at", coordinates) # elif j["result"]["eventTypeId"] == "MISSED_SHOT": # if j["players"][0]["player"]["id"] == 8471214: # coordinates = convertCoord(gamefeed, venue, j["about"]["period"], float(j["coordinates"]["x"]), float(j["coordinates"]["y"])) # jsonshotlist.append({"type":j["result"]["eventTypeId"],"time":eventtime,"coordinates":coordinates}) # # print("Event", i, j["players"][0]["player"]["fullName"], j["result"]["event"], eventtime, "at", coordinates) # elif j["result"]["eventTypeId"] == "BLOCKED_SHOT": # if j["players"][1]["player"]["id"] == 8471214: # coordinates = convertCoord(gamefeed, venue, j["about"]["period"], float(j["coordinates"]["x"]), float(j["coordinates"]["y"])) # jsonshotlist.append({"type":j["result"]["eventTypeId"],"time":eventtime,"coordinates":coordinates}) # # print("Event", i, j["players"][1]["player"]["fullName"], j["result"]["event"], eventtime, "at", coordinates) # else: # pass # else: # pass # else: # pass shifts = [] ovishifts = [] for i, j in pandashifts.iterrows(): shift = parseShift(j["Period"], j["Start"], j["End"]) data = {} data["player"] = j["Player"] data["team"] = j["Team"] data["start"] = shift[0] data["end"] = shift[1] shifts.append(data) for i in shifts: if i["player"] == "ALEX OVECHKIN": ovishifts.append(i) oppsOnIceWithOvi = [] capsOnIceWithOvi = [] for i, a in enumerate(ovishifts): for b in shifts: if b in capsOnIceWithOvi: continue elif b in oppsOnIceWithOvi: continue else: if b["start"] < a["start"] and b["end"] > a["end"]: if b["team"] == "WSH": if b["start"] <= a["start"]: start = a["start"] else: start = b["start"] if b["end"] >= a["end"]: end = a["end"] else: end = b["end"] capsOnIceWithOvi.append({ "player": b["player"], "team": b["team"], "start": start, "end": end }) else: if b["start"] <= a["start"]: start = a["start"] else: start = b["start"] if b["end"] >= a["end"]: end = a["end"] else: end = b["end"] oppsOnIceWithOvi.append({ "player": b["player"], "team": b["team"], "start": start, "end": end }) continue if b["start"] >= a["start"] and b["start"] < a["end"]: if b["team"] == "WSH": if b["start"] <= a["start"]: start = a["start"] else: start = b["start"] if b["end"] >= a["end"]: end = a["end"] else: end = b["end"] capsOnIceWithOvi.append({ "player": b["player"], "team": b["team"], "start": start, "end": end }) else: if b["start"] <= a["start"]: start = a["start"] else: start = b["start"] if b["end"] >= a["end"]: end = a["end"] else: end = b["end"] oppsOnIceWithOvi.append({ "player": b["player"], "team": b["team"], "start": start, "end": end }) continue if b["end"] <= a["end"] and b["end"] > a["start"]: if b["team"] == "WSH": if b["start"] <= a["start"]: start = a["start"] else: start = b["start"] if b["end"] >= a["end"]: end = a["end"] else: end = b["end"] capsOnIceWithOvi.append({ "player": b["player"], "team": b["team"], "start": start, "end": end }) else: if b["start"] <= a["start"]: start = a["start"] else: start = b["start"] if b["end"] >= a["end"]: end = a["end"] else: end = b["end"] oppsOnIceWithOvi.append({ "player": b["player"], "team": b["team"], "start": start, "end": end }) continue strengthseconds = [] pandastrengthshotlist = [] for c in range(0, 3900): # print(c,"seconds") totCapsOnIce = 0 totOppsOnIce = 0 for d in capsOnIceWithOvi: if d["player"] in capsgoalies: pass else: if d["start"] <= c < d["end"]: totCapsOnIce += 1 # print(d["player"]) for e in oppsOnIceWithOvi: if e["player"] in oppsgoalies: pass else: if e["start"] <= c < e["end"]: totOppsOnIce += 1 # print(e["player"]) if totCapsOnIce > 6: print(c, "seconds", totCapsOnIce, "Caps On Ice") totCapsOnIce = 5 if totOppsOnIce > 6: print(c, "seconds", totOppsOnIce, "Opponents On Ice") totOppsOnIce = 5 strengthseconds.append(str(totCapsOnIce) + "v" + str(totOppsOnIce)) # print(str(totCapsOnIce) + "v" + str(totOppsOnIce)) for f in pandashotlist: strength = strengthseconds[f["time"]] if strength == "0v0": strength = strengthseconds[f["time"] - 1] event = { "type": f["type"], "time": f["time"], "coordinates": f["coordinates"], "strength": strength } pandastrengthshotlist.append(event) strengths = {} for item in strengthseconds: if item in strengths.keys(): strengths[item] += 1 else: strengths[item] = 1 return {"time": strengths, "events": pandastrengthshotlist}
def lambda_handler(event, context): LAMBDA_GENERATOR = os.environ.get("LAMBDA_GENERATOR") IS_SNS_TRIGGER = bool(event.get("Records")) game_id_dict = get_game_id(event) game_id_status = game_id_dict["status"] if IS_SNS_TRIGGER: msg = event['Records'][0]['Sns']['Message'] # Convert Message back to JSON Object msg = json.loads(msg) game_id = msg['gamePk'] period = msg['play']['about']['period'] goals = msg['play']['about']['goals'] home_score = goals['home'] away_score = goals['away'] else: # Get scores directly from event payload home_score = event.get("home_score") away_score = event.get("away_score") if not game_id_status: logging.error(game_id_dict["msg"]) return {"status": False, "msg": game_id_dict["msg"]} # If status is True, set the game_id variable game_id = game_id_dict["game_id"] # Check that the event period is greater than the last processed period is_event_newer = is_event_period_newer(game_id, period) if not is_event_newer: logging.error( "The event received for %s is not newer than the last " "event recorded in the database - skip this record.", game_id) return { 'status': 409, 'body': 'A shotmap was already produced for this event.' } # If all of the above checks pass, scrape the game. scraped_data = hockey_scraper.scrape_games([game_id], False, data_format="Pandas") pbp = scraped_data.get("pbp") # fmt: off cols_to_drop = [ 'awayPlayer1', 'awayPlayer1_id', 'awayPlayer2', 'awayPlayer2_id', 'awayPlayer3', 'awayPlayer3_id', 'awayPlayer4', 'awayPlayer4_id', 'awayPlayer5', 'awayPlayer5_id', 'awayPlayer6', 'awayPlayer6_id', 'homePlayer1', 'homePlayer1_id', 'homePlayer2', 'homePlayer2_id', 'homePlayer3', 'homePlayer3_id', 'homePlayer4', 'homePlayer4_id', 'homePlayer5', 'homePlayer5_id', 'homePlayer6', 'homePlayer6_id', 'Description', 'Home_Coach', 'Away_Coach' ] # fmt: on pbp = pbp.drop(cols_to_drop, axis=1) pbp.columns = map(str.lower, pbp.columns) pbp_json = pbp.to_json() payload = { "pbp_json": pbp_json, "game_id": game_id, "testing": TESTING, "home_score": home_score, "away_score": away_score } small_payload = { "game_id": game_id, "testing": TESTING, "home_score": home_score, "away_score": away_score } logging.info( "Scraping completed. Triggering the generator & twitter Lambda.") invoke_response = lambda_client.invoke(FunctionName=LAMBDA_GENERATOR, InvocationType="Event", Payload=json.dumps(payload)) print(invoke_response) return {'body': small_payload}
def dataScrape(gameID, venue, gamefeed): if venue == "home": print("Ovechkin played at home") elif venue == "away": print("Ovechkin played on the road") else: print("venue error") shotsgoals = [] fenwickgoals = [] # corsigoals = [] shots = [] fenwick = [] # corsi = [] evshotsgoals = [] evfenwickgoals = [] # evcorsigoals = [] evshots = [] evfenwick = [] # evcorsi = [] ppshotsgoals = [] ppfenwickgoals = [] # ppcorsigoals = [] ppshots = [] ppfenwick = [] # ppcorsi = [] # print(gamefeed) scrape = hockey_scraper.scrape_games([gameID], False, data_format="Pandas") bigData = scrape["pbp"] eventlist = ["GOAL", "SHOT", "MISS"] shotdata = bigData.loc[bigData["Event"].isin(eventlist)] roshotdata = shotdata.loc[shotdata["Period"] <= 4] ovishotdata = roshotdata.loc[roshotdata["p1_ID"] == 8471214] # print(ovishotdata) for i, j in ovishotdata.iterrows(): coordinates = convertCoord(gamefeed, venue, j["Period"], j["xC"], j["yC"]) # print(i, j["Description"], newcoord) strength = j["Strength"] strsplit = strength.split("x") if venue == "home": if int(strsplit[0]) >= 5 and int(strsplit[1]) >= 5: if j["Event"] == "GOAL": shotsgoals.append(1) fenwickgoals.append(1) # corsigoals.append(1) shots.append(coordinates) fenwick.append(coordinates) # corsi.append(coordinates) evshotsgoals.append(1) evfenwickgoals.append(1) # evcorsigoals.append(1) evshots.append(coordinates) evfenwick.append(coordinates) # evcorsi.append(coordinates) elif j["Event"] == "SHOT": shotsgoals.append(0) fenwickgoals.append(0) # corsigoals.append(0) shots.append(coordinates) fenwick.append(coordinates) # corsi.append(coordinates) evshotsgoals.append(0) evfenwickgoals.append(0) # evcorsigoals.append(0) evshots.append(coordinates) evfenwick.append(coordinates) # evcorsi.append(coordinates) elif j["Event"] == "MISS": fenwickgoals.append(0) # corsigoals.append(0) fenwick.append(coordinates) # corsi.append(coordinates) evfenwickgoals.append(0) # evcorsigoals.append(0) evfenwick.append(coordinates) # evcorsi.append(coordinates) elif int(strsplit[1]) == int(strsplit[0]): if j["Event"] == "GOAL": shotsgoals.append(1) fenwickgoals.append(1) # corsigoals.append(1) shots.append(coordinates) fenwick.append(coordinates) # corsi.append(coordinates) evshotsgoals.append(1) evfenwickgoals.append(1) # evcorsigoals.append(1) evshots.append(coordinates) evfenwick.append(coordinates) # evcorsi.append(coordinates) elif j["Event"] == "SHOT": shotsgoals.append(0) fenwickgoals.append(0) # corsigoals.append(0) shots.append(coordinates) fenwick.append(coordinates) # corsi.append(coordinates) evshotsgoals.append(0) evfenwickgoals.append(0) # evcorsigoals.append(0) evshots.append(coordinates) evfenwick.append(coordinates) # evcorsi.append(coordinates) elif j["Event"] == "MISS": fenwickgoals.append(0) # corsigoals.append(0) fenwick.append(coordinates) # corsi.append(coordinates) evfenwickgoals.append(0) # evcorsigoals.append(0) evfenwick.append(coordinates) # evcorsi.append(coordinates) elif int(strsplit[1]) < int(strsplit[0]): if j["Event"] == "GOAL": shotsgoals.append(1) fenwickgoals.append(1) # corsigoals.append(1) shots.append(coordinates) fenwick.append(coordinates) # corsi.append(coordinates) ppshotsgoals.append(1) ppfenwickgoals.append(1) # ppcorsigoals.append(1) ppshots.append(coordinates) ppfenwick.append(coordinates) # ppcorsi.append(coordinates) elif j["Event"] == "SHOT": shotsgoals.append(0) fenwickgoals.append(0) # corsigoals.append(0) shots.append(coordinates) fenwick.append(coordinates) # corsi.append(coordinates) ppshotsgoals.append(0) ppfenwickgoals.append(0) # ppcorsigoals.append(0) ppshots.append(coordinates) ppfenwick.append(coordinates) # ppcorsi.append(coordinates) elif j["Event"] == "MISS": fenwickgoals.append(0) # corsigoals.append(0) fenwick.append(coordinates) # corsi.append(coordinates) ppfenwickgoals.append(0) # ppcorsigoals.append(0) ppfenwick.append(coordinates) # ppcorsi.append(coordinates) else: pass elif venue == "away": if int(strsplit[0]) >= 5 and int(strsplit[1]) >= 5: if j["Event"] == "GOAL": shotsgoals.append(1) fenwickgoals.append(1) # corsigoals.append(1) shots.append(coordinates) fenwick.append(coordinates) # corsi.append(coordinates) evshotsgoals.append(1) evfenwickgoals.append(1) # evcorsigoals.append(1) evshots.append(coordinates) evfenwick.append(coordinates) # evcorsi.append(coordinates) elif j["Event"] == "SHOT": shotsgoals.append(0) fenwickgoals.append(0) # corsigoals.append(0) shots.append(coordinates) fenwick.append(coordinates) # corsi.append(coordinates) evshotsgoals.append(0) evfenwickgoals.append(0) # evcorsigoals.append(0) evshots.append(coordinates) evfenwick.append(coordinates) # evcorsi.append(coordinates) elif j["Event"] == "MISS": fenwickgoals.append(0) # corsigoals.append(0) fenwick.append(coordinates) # corsi.append(coordinates) evfenwickgoals.append(0) # evcorsigoals.append(0) evfenwick.append(coordinates) # evcorsi.append(coordinates) elif int(strsplit[0]) == int(strsplit[1]): if j["Event"] == "GOAL": shotsgoals.append(1) fenwickgoals.append(1) # corsigoals.append(1) shots.append(coordinates) fenwick.append(coordinates) # corsi.append(coordinates) evshotsgoals.append(1) evfenwickgoals.append(1) # evcorsigoals.append(1) evshots.append(coordinates) evfenwick.append(coordinates) # evcorsi.append(coordinates) elif j["Event"] == "SHOT": shotsgoals.append(0) fenwickgoals.append(0) # corsigoals.append(0) shots.append(coordinates) fenwick.append(coordinates) # corsi.append(coordinates) evshotsgoals.append(0) evfenwickgoals.append(0) # evcorsigoals.append(0) evshots.append(coordinates) evfenwick.append(coordinates) # evcorsi.append(coordinates) elif j["Event"] == "MISS": fenwickgoals.append(0) # corsigoals.append(0) fenwick.append(coordinates) # corsi.append(coordinates) evfenwickgoals.append(0) # evcorsigoals.append(0) evfenwick.append(coordinates) # evcorsi.append(coordinates) elif int(strsplit[0]) < int(strsplit[1]): if j["Event"] == "GOAL": shotsgoals.append(1) fenwickgoals.append(1) # corsigoals.append(1) shots.append(coordinates) fenwick.append(coordinates) # corsi.append(coordinates) ppshotsgoals.append(1) ppfenwickgoals.append(1) # ppcorsigoals.append(1) ppshots.append(coordinates) ppfenwick.append(coordinates) # ppcorsi.append(coordinates) elif j["Event"] == "SHOT": shotsgoals.append(0) fenwickgoals.append(0) # corsigoals.append(0) shots.append(coordinates) fenwick.append(coordinates) # corsi.append(coordinates) ppshotsgoals.append(0) ppfenwickgoals.append(0) # ppcorsigoals.append(0) ppshots.append(coordinates) ppfenwick.append(coordinates) # ppcorsi.append(coordinates) elif j["Event"] == "MISS": fenwickgoals.append(0) # corsigoals.append(0) fenwick.append(coordinates) # corsi.append(coordinates) ppfenwickgoals.append(0) # ppcorsigoals.append(0) ppfenwick.append(coordinates) # ppcorsi.append(coordinates) else: pass avgx = 0 totalx = 0 for i in range(0, len(fenwick)): totalx = totalx + fenwick[i][0] if len(fenwick) > 0: avgx = totalx / len(fenwick) if avgx > 100: # print("-------------------- INVERT HERE --------------------") for fen in range(0, len(fenwick)): fenwick[fen][0] = -fenwick[fen][0] + 178 fenwick[fen][1] = -fenwick[fen][1] print(shots) print(fenwick) print(evshots) print(evfenwick) print(ppshots) print(ppfenwick) # print("---------------- INVERSION COMPLETE ----------------") return [ shotsgoals, fenwickgoals, shots, fenwick, evshotsgoals, evfenwickgoals, evshots, evfenwick, ppshotsgoals, ppfenwickgoals, ppshots, ppfenwick ]
import hockey_scraper hockey_scraper.scrape_games([2015020031], True)