def scrape_pbp(
game_id, date, roster, game_json, players, teams, espn_id=None, html_df=None
):
"""
Automatically scrapes the json and html, if the json is empty the html picks up some of the slack and the espn
xml is also scraped for coordinates.
:param game_id: json game id
:param date: date of game
:param roster: list of players in pre game roster
:param game_json: json pbp for game
:param players: dict of players
:param teams: dict of teams
:param espn_id: Game Id for the espn game. Only provided when live scraping
:param html_df: Can provide DataFrame for html. Only done for live-scraping
:return: DataFrame with info or None if it fails
"""
# Coordinates are only available in json from 2010 onwards
# Note: This doesn't work as intended for second half of 2009 season...it still works just takes slightly longer
if int(str(game_id)[:4]) >= 2010:
json_df = json_pbp.parse_json(game_json, game_id)
if json_df is None:
return None # Means there was an error parsing
if_json = True if len(game_json["liveData"]["plays"]["allPlays"]) > 0 else False
else:
if_json = False
# Only scrape if nothing provided
if not isinstance(html_df, pd.DataFrame):
html_df = html_pbp.scrape_game(game_id, players, teams)
# Got nothing if it isn't there
if html_df is None:
return None
# Check if the json is missing the plays...if it is scrape ESPN for the coordinates
if not if_json:
espn_df = espn_pbp.scrape_game(
date, teams["Home"], teams["Away"], game_id=espn_id
)
game_df = combine_espn_html_pbp(
html_df, espn_df, str(game_id), date, teams["Away"], teams["Home"]
)
# Sometimes espn is corrupted so can't get coordinates
if espn_df is None or espn_df.empty:
missing_coords.extend([[game_id, date]])
else:
game_df = combine_html_json_pbp(json_df, html_df, str(game_id), date)
if game_df is not None:
game_df["Home_Coach"] = roster["head_coaches"]["Home"]
game_df["Away_Coach"] = roster["head_coaches"]["Away"]
return game_df
def scrape_pbp(game_id,
date,
roster,
game_json,
players,
teams,
espn_id=None,
html_df=None):
"""
Scrape the Pbp info. The HTML is always scraped.
The Json is parse whe season >= 2010 and there are plays. Otherwise ESPN is gotten to supplement
the HTML with coordinate.
The espn_id and the html data can be fed as keyword argument to speed up execution. This is used by
the live game scraping class.
:param game_id: json game id
:param date: date of game
:param roster: list of players in pre game roster
:param game_json: json pbp for game
:param players: dict of players
:param teams: dict of teams
:param espn_id: Game Id for the espn game. Only provided when live scraping
:param html_df: Can provide DataFrame for html. Only done for live-scraping
:return: DataFrame with info or None if it fails
"""
# Coordinates are only available in json from 2010 onwards
if int(str(game_id)[:4]) >= 2010 and len(
game_json['liveData']['plays']['allPlays']) > 0:
json_df = json_pbp.parse_json(game_json, game_id)
else:
json_df = None
# For live sometimes the json lags the html so if given we don't bother
if not isinstance(html_df, pd.DataFrame):
html_df = html_pbp.scrape_game(game_id, players, teams)
if html_df is None or html_df.empty:
return None
# Check if the json is missing the plays...if it is scrape ESPN for the coordinates
if json_df is None or json_df.empty:
espn_df = espn_pbp.scrape_game(date,
teams['Home'],
teams['Away'],
game_id=espn_id)
game_df = combine_espn_html_pbp(html_df, espn_df, str(game_id), date,
teams['Away'], teams['Home'])
# Sometimes espn is corrupted so can't get coordinates
if espn_df is None or espn_df.empty:
missing_coords.extend([[game_id, date]])
else:
game_df = combine_html_json_pbp(json_df, html_df, str(game_id), date)
if game_df is not None:
game_df['Home_Coach'] = roster['head_coaches']['Home']
game_df['Away_Coach'] = roster['head_coaches']['Away']
return game_df