def fetch_raw_data(self):
for btype in self.box_types[:-1]:
full_url = self.base_url + self.endpoint.format(btype=btype)
rawdata = get_json_response(full_url, params=self.params)
self.raw_data[btype] = rawdata
full_url = self.base_url + "hustlestatsboxscore"
rawdata = get_json_response(full_url, params=self.params)
self.raw_data['hustle'] = rawdata
def create_play_by_play_for_game(game):
log.debug("Creating Play By Play Events for Game " + str(game.game_id))
pbp_events = []
url = (NBA_BASE_URL + PBP_URL).format(game_id=game.game_id,
season=make_season_str(game.season),
season_type='foo')
json_data = get_json_response(url)
pbp_data = json_data['resultSets'][0]
headers = pbp_data['headers']
for pbp_event in pbp_data['rowSet']:
temp_dict = dict(zip(headers, pbp_event))
temp_dict['GAME'] = game
# TODO: Write a method that will look for a player and if it doesn't find, creates it.
# TODO: The method should then return the player object
if temp_dict['PLAYER1_NAME']:
player1 = Player.objects.filter(
player_id=temp_dict['PLAYER1_ID']).first()
if not player1:
player1 = Player(player_id=temp_dict['PLAYER1_ID'],
display_first_last=temp_dict['PLAYER1_NAME'])
player1.save()
temp_dict['PLAYER1'] = player1
if temp_dict['PLAYER1_TEAM_ID']:
temp_dict['PLAYER1_TEAM'] = Team.objects.filter(
team_id=temp_dict['PLAYER1_TEAM_ID']).first()
if temp_dict['PLAYER2_NAME']:
player2 = Player.objects.filter(
player_id=temp_dict['PLAYER2_ID']).first()
if not player2:
player2 = Player(player_id=temp_dict['PLAYER2_ID'],
display_first_last=temp_dict['PLAYER2_NAME'])
player2.save()
temp_dict['PLAYER2'] = player2
if temp_dict['PLAYER2_TEAM_ID']:
temp_dict['PLAYER2_TEAM'] = Team.objects.filter(
team_id=temp_dict['PLAYER2_TEAM_ID']).first()
if temp_dict['PLAYER3_NAME']:
player3 = Player.objects.filter(
player_id=temp_dict['PLAYER3_ID']).first()
if not player3:
player3 = Player(player_id=temp_dict['PLAYER3_ID'],
display_first_last=temp_dict['PLAYER1_NAME'])
player3.save()
temp_dict['PLAYER3'] = player3
if temp_dict['PLAYER3_TEAM_ID']:
temp_dict['PLAYER3_TEAM'] = Team.objects.filter(
team_id=temp_dict['PLAYER3_TEAM_ID']).first()
if temp_dict['SCOREMARGIN'] == 'TIE':
temp_dict['SCOREMARGIN'] = 0
pbp_create_data = convert_dict_keys_to_lowercase(temp_dict)
pbp_obj = PlayByPlayEvent(**pbp_create_data)
pbp_events.append(pbp_obj)
return pbp_events
def get_player_info_from_web(player_id):
url = (NBA_BASE_URL + PLAYER_INFO_URL).format(player_id=str(player_id),
season_type="Regular+Season")
log.debug("Player URL: " + url)
data = get_json_response(url)
# This goofy thing returns a list containing the relevant player information
headers = data['resultSets'][0]['headers']
info = data['resultSets'][0]['rowSet'][0]
data_dict = dict(zip(headers, info))
return data_dict
def get_team_info_from_web(team_id):
# season should probably be passed or derived from current date
url = NBA_BASE_URL + TEAM_INFO_BASE_URL.format(id=str(team_id),
season="2016-17")
log.debug("Team URL: " + url)
data = get_json_response(url)
headers = data['resultSets'][0]['headers']
info = data['resultSets'][0]['rowSet'][0]
data_dict = dict(zip(headers, info))
return data_dict
def get_player_shotchart(player, season, season_type="Regular Season"):
params = SHOTCHART_PARAMS
params['PlayerID'] = player.player_id
params['Season'] = season
params['SeasonType'] = season_type
url = NBA_BASE_URL + 'shotchartdetail'
# log.debug("URL: " + url)
# log.debug("Params: \n" + str(params))
json_data = get_json_response(url, params=params)
# log.debug("JSON Data: \n" + str(json_data))
relevant_data = json_data['resultSets'][0]
return relevant_data
def create_update_all_seasons_for_player(player, year=None):
player_seasons = []
url = NBA_BASE_URL + PLAYER_CAREER_STATS_ENDPOINT
parms = PLAYER_CAREER_STATS_PARAMS
parms['PlayerID'] = player.player_id
# Removing Per36 & PerGame cut down the requests by 2/3
per_mode = "Totals"
parms['PerMode'] = per_mode
json_data = get_json_response(url, params=parms)
result_sets = json_data['resultSets']
for rset in result_sets:
season_type = rset['name']
if not any(ignore in season_type for ignore in IGNORE_SEASON_TYPES):
headers = rset['headers']
rows = rset['rowSet']
team = None
for row in rows:
data = dict(zip(headers, row))
# log.debug("Data: " + str(data))
dteam_key = "TEAM_ID" if "TEAM_ID" in data else "Team_ID"
if team is None or data[dteam_key] != team.team_id:
try:
team = Team.objects.get(team_id=data[dteam_key])
except Exception as e:
log.debug(("ELEPHANT", data))
raise e
conv_data = convert_dict_keys_to_lowercase(data, override_list=['LEAGUE_ID'])
conv_data['season_id'] = make_season_int(conv_data.get('season_id', 0))
# year=None -> update/create all seasons. Always update career.
if year is not None and conv_data['season_id'] not in [0, year]:
continue
conv_data['season_type'] = season_type
conv_data['player'] = player
conv_data['team'] = team
conv_data['per_mode'] = per_mode
# log.debug("Converted Data: " + str(conv_data))
filters = make_unique_filter_dict(PlayerSeason, conv_data)
season, created = PlayerSeason.objects.update_or_create(**filters,
defaults=conv_data)
if created:
log.debug(("Created a new season: ", filters))
player_seasons.append(season)
return player_seasons
def create_update_team_seasons(team, season):
log.debug("Seeding Team Seasons")
# I am sure there are others we need to exclude
url = NBA_BASE_URL + TEAM_SEASONS_ENDPOINT
parms = TEAM_SEASONS_PARMS
log.debug(team.city + " " + team.name)
parms['TeamID'] = team.team_id
# TODO: Something weird is going on with playoffs
# We're getting the exact same stats as reg season, even though I checked the request &
# It seems correct. When I look via browser, the stats seem to be correct.
for season_type in ["Regular Season", "Playoffs"]:
log.debug(season_type)
parms['SeasonType'] = season_type
for permode in ["PerGame", "Totals"]:
log.debug(permode)
parms['PerMode'] = permode
try:
json_data = get_json_response(url, params=parms)
except Exception as e:
log.debug(("PARMS ", parms))
log.debug(("URL ", url))
log.exception(e)
raise e
process_team_season_json(team, json_data, season)
def fetch_raw_data(self):
self.raw_data = get_json_response(self.full_url, params=self.params)
def _fetch_raw_data(self, scoreboard_date):
date_parm = scoreboard_date.strftime("%m/%d/%Y")
self.params['GameDate'] = date_parm
self.date = scoreboard_date
self.raw_data = get_json_response(self.full_url, self.params)
def create_update_all_splits_for_entity(entity, year=None, season_type="Regular Season"):
splits = {'Base': [], 'Advanced': [], 'Misc': [],
'Scoring': [], 'Usage': [], 'Shooting': []}
parms = GENERAL_SPLITS_PARMS
if isinstance(entity, Player):
parms['PlayerID'] = ent_nba_id = entity.player_id
endpoint = PLAYER_GENERAL_SPLITS_ENDPOINT
shooting_endpoint = PLAYER_SHOOTING_SPLITS_ENDPOINT
elif isinstance(entity, Team):
parms['TeamID'] = ent_nba_id = entity.team_id
endpoint = TEAM_GEN_SPLITS_ENDPOINT
shooting_endpoint = TEAM_SHOOTING_SPLITS_ENDPOINT
else:
raise Exception("What are you doing. This function requires a Player or Team.")
# teams default to 1996-2015-16 season
first_year = getattr(entity, 'from_year', None) or 1996
first_year = 1996 if first_year < 1996 else first_year
final_year = getattr(entity, 'to_year', None) or date.today().year
if year is not None:
if year < first_year or year > final_year:
raise ValueError("You passed an invalid year: {yr}. "
"Year must be between {first} and {last} "
"for {ent}.".format(yr=year, first=first_year,
last=final_year, ent=str(entity)))
else:
first_year = year
final_year = year + 1
# Need to loop over every measure_type, per_mode, season, season_type
# Could cut this down to Totals & Per100, then calculate everything else.
# That cuts the requests in half. Could also jut get box scores, but I think that would
# Make it tough (impossible) to get/do per possession data.
for per_mode in ['Totals', 'Per100Possessions']:
parms['PerMode'] = per_mode
parms['SeasonType'] = season_type
for cur_year in range(first_year, final_year):
parms['Season'] = make_season_str(cur_year)
# This is probably the most effective place to make a cut,
# But I'm not sure if it can be done.
for measure_type in ['Base', 'Advanced', 'Misc', 'Scoring', 'Usage']:
parms['MeasureType'] = measure_type
log.debug((ent_nba_id, cur_year, season_type, per_mode, measure_type))
url = NBA_BASE_URL + endpoint
# Temp. for Hornets
try:
# This is where like 90% of the time is being spent
data = get_json_response(url, parms)
except Exception as e:
log.exception(e)
continue
entity_splits = create_update_split_from_raw_json(entity, data)
splits[measure_type] += entity_splits
url = NBA_BASE_URL + shooting_endpoint
# I don't think this actually matters
parms['MeasureType'] = "Base"
data = get_json_response(url, parms)
log.debug((ent_nba_id, cur_year, season_type, per_mode, "Shooting"))
shooting_splits = create_update_split_from_raw_json(entity, data)
splits['Shooting'] += shooting_splits
return splits
def create_box_scores_for_game(game,
url_suffix,
season_type="Regular+Season",
box_type="Traditional"):
player_box_scores = []
team_box_scores = []
url = (NBA_BASE_URL + url_suffix).format(game_id=game.game_id,
season=make_season_str(
game.season),
season_type=season_type)
data = get_json_response(url)
if box_type == "Hustle":
hustle_status = data['resultSets'][0]['rowSet'][0][1]
if not hustle_status:
player_stats = None
else:
player_stats = data['resultSets'][1]
else:
player_stats = data['resultSets'][0]
if player_stats:
player_headers = player_stats['headers']
if box_type == "Hustle":
team_stats = data['resultSets'][2]
else:
team_stats = data['resultSets'][1]
team_headers = team_stats['headers']
for p in player_stats['rowSet']:
temp_dict = dict(zip(player_headers, p))
temp_dict['GAME'] = game
temp_dict['TEAM'] = Team.objects.filter(
team_id=temp_dict['TEAM_ID']).first()
player = Player.objects.filter(
player_id=temp_dict['PLAYER_ID']).first()
if not player:
# TODO: Properly handle the situation where the given player doesn't exist yet
log.debug("Player id and name about to look up: %s %s" %
(temp_dict['PLAYER_ID'], temp_dict['PLAYER_NAME']))
# Laziest possible player creation...
# should probably write a script that goes through the db and
# attempts to fill in missing player data
player = Player(display_first_last=temp_dict['PLAYER_NAME'],
player_id=temp_dict['PLAYER_ID'],
team=temp_dict['TEAM'])
player.save()
temp_dict['PLAYER'] = player
if box_type == "Tracking":
converted_dict = convert_tracking_dict_to_nbapex_fields(
temp_dict, player_flg=True)
else:
converted_dict = temp_dict
p_box_data = convert_dict_keys_to_lowercase(converted_dict)
if box_type == 'Hustle':
p_box_data['minutes'] = convert_min_sec_to_float(
p_box_data['minutes'])
else:
p_box_data['min'] = convert_min_sec_to_float(p_box_data['min'])
pbox = create_correct_type_of_player_box_score(
p_box_data, box_type)
player_box_scores.append(pbox)
for t in team_stats['rowSet']:
temp_dict = dict(zip(team_headers, t))
temp_dict['GAME'] = game
temp_dict['TEAM'] = Team.objects.filter(
team_id=temp_dict['TEAM_ID']).first()
if box_type == 'Tracking':
converted_dict = convert_tracking_dict_to_nbapex_fields(
temp_dict, player_flg=False)
else:
converted_dict = temp_dict
t_box_data = convert_dict_keys_to_lowercase(converted_dict)
if box_type == "Hustle":
t_box_data['minutes'] = convert_min_sec_to_float(
t_box_data['minutes'])
else:
t_box_data['min'] = convert_min_sec_to_float(t_box_data['min'])
tbox = create_correct_type_of_team_box_score(t_box_data, box_type)
team_box_scores.append(tbox)
else:
log.debug("Box scores of type {bt} not available for game {gm}".format(
bt=box_type, gm=game.game_id))
ret_dict = {'players': player_box_scores, 'teams': team_box_scores}
return ret_dict
def create_games(game_info):
headers = game_info['resultSets'][0]['headers']
list_game_info = game_info['resultSets'][0]['rowSet']
games = []
line_scores = []
all_box_scores = []
pbp_events = []
other_stats = []
official_xrefs = []
for info in list_game_info:
info[0] = convert_datetime_string_to_date_instance(info[0])
temp_dict = dict(zip(headers, info))
existing_game = Game.objects.filter(game_id=temp_dict['GAME_ID'])
if existing_game.exists():
log.debug("Game {game} already exists!".format(
game=existing_game.first()))
log.debug("Avoided creating a new one. Moving on...")
else:
temp_dict['HOME_TEAM'] = Team.objects.get(
team_id=temp_dict['HOME_TEAM_ID'])
temp_dict['VISITOR_TEAM'] = Team.objects.get(
team_id=temp_dict['VISITOR_TEAM_ID'])
url = (NBA_BASE_URL +
SUMMARY_URL).format(game_id=temp_dict['GAME_ID'])
ancillary_game_data = get_json_response(url)
ancillary_game_data = ancillary_game_data['resultSets']
# TODO: I'm sure we are going to get index OOB Exception here at some point.
other_stats_data = ancillary_game_data[1]
officials_data = ancillary_game_data[2]
game_info_data = dict(
zip(ancillary_game_data[4]['headers'],
ancillary_game_data[4]['rowSet'][0]))
line_score_data = ancillary_game_data[5]
temp_dict['ATTENDANCE'] = game_info_data['ATTENDANCE']
temp_dict['GAME_TIME'] = convert_colon_tstamp_to_duration(
game_info_data['GAME_TIME'])
game_data = convert_dict_keys_to_lowercase(temp_dict, isgame=True)
game = Game(**game_data)
# Have to save the game now so that other object that reference can point to its id.
# I'd like to find a way around this, but for now I don't know of one.
game.save()
games.append(game)
line_scores += create_line_scores_for_game(game, line_score_data)
# This is a dict of dicts containing a dictionary for each type of box score.
# Each of those 5 dicts contains team and player box scores.
all_box_scores_dict = create_all_box_scores_for_game(game)
all_box_scores.append(all_box_scores_dict)
pbp_events += create_play_by_play_for_game(game)
other_stats += create_other_stats_for_game(game, other_stats_data)
official_xrefs += create_game_official_xrefs(game, officials_data)
ret_dict = {
'games': games,
'line_scores': line_scores,
'box_scores': all_box_scores,
'pbp_events': pbp_events,
'other_stats': other_stats,
'official_xrefs': official_xrefs
}
return ret_dict
def fetch_raw_data(self,
season=None,
season_type=REGULAR_SEASON,
pt_measure_type=SPEED_DISTANCE,
per_mode=TOTALS,
group_set="Overall",
group_value=None):
self.raw_data = []
self.pt_measure_type = pt_measure_type
if self.pt_measure_type in ["CatchShoot", "PullUpShot"]:
middle = "ShotType"
elif "Touch" in self.pt_measure_type or "Drive" in self.pt_measure_type:
middle = "Touch"
else:
middle = self.pt_measure_type
self.model = get_model(prefix="Player",
middle=middle,
suffix="Tracking")
# Silly way to do this, I know
url = self.base_url + self.endpoint
if season is None:
season = make_season_str(determine_season_for_date(date.today()))
if group_set == "Overall" and group_value is None:
group_value = season
self.params['Season'] = str(season)
self.params['SeasonType'] = season_type
self.params['PerMode'] = per_mode
self.params['PtMeasureType'] = pt_measure_type
if group_set != "Overall":
self.params[group_set] = group_value
if group_set != "Month":
group_value = GROUP_VALUES[group_value]
log.debug(("Fetching Player Stats: ", season, season_type,
pt_measure_type, per_mode, group_set, group_value))
raw_json = get_json_response(url, self.params)
rset = raw_json['resultSets'][0]
player_rows = dictify(result_set=rset)
processed_rows = []
for row in player_rows:
row['PLAYER'] = Player.objects.get(player_id=row['PLAYER_ID'])
row['GROUP_SET'] = group_set
row['SEASON'] = season
row['SEASON_TYPE'] = season_type
row['PT_MEASURE_TYPE'] = pt_measure_type
row['PER_MODE'] = per_mode
if group_set == "Month":
row['GROUP_VALUE'] = REVERSE_MONTH_MAP[group_value]
else:
row['GROUP_VALUE'] = group_value
if self.pt_measure_type in ["CatchShoot", "PullUpShot"]:
row = convert_shot_type_dict_to_apex(row)
elif "Touch" in self.pt_measure_type or "Drive" in self.pt_measure_type:
row = convert_touch_dict_to_apex(row)
proc_row = convert_dict_keys_to_lowercase(
row, override_list=['GROUP_SET', 'GROUP_VALUE'])
processed_rows.append(proc_row)
self.raw_data = processed_rows
