def create_year_of_data_directories(year):
is_valid_year(year)
game_date_info = get_first_and_last_game_of_year(year)
start_day = game_date_info['start_day']
start_month = game_date_info['start_month']
end_day = game_date_info['end_day']
end_month = game_date_info['end_month']
all_star_day = game_date_info['all_star_day']
os.chdir('data')
year_dir_path = str(os.getcwd()) + '/' + str(year)
print year_dir_path
if not os.path.exists(year_dir_path):
os.makedirs(year_dir_path)
os.chdir(str(year))
month_dir_path = str(os.getcwd()) + '/' + str(
convert_month_int_to_string(start_month))
print month_dir_path
first_month = True
for month in range(start_month, end_month):
month_as_string = convert_month_int_to_string(month)
month_dir_path = str(os.getcwd()) + '/' + month_as_string
if not os.path.exists(month_dir_path):
os.makedirs(month_dir_path)
if first_month:
create_month_of_data_directories(month, start_day, 0, 0)
first_month = False
elif month == 7:
create_month_of_data_directories(month, 0, 0, all_star_day)
else:
create_month_of_data_directories(month, 0, 0, 0)
# Create last month
month_as_string = convert_month_int_to_string(end_month)
month_dir_path = str(os.getcwd()) + '/' + month_as_string
if not os.path.exists(month_dir_path):
os.makedirs(month_dir_path)
create_month_of_data_directories(end_month, 0, end_day, 0)
os.chdir('..')
os.chdir('..')
def download_game_data_for_today(url, year, month, day):
base_file_name = get_scoreboard_file(year, month, day)
base_scoreboard_directory = str(
str(os.getcwd()) + '/data/' + str(year) + '/' +
convert_month_int_to_string(month) + '/' +
convert_number_to_string(day) + '/')
print base_scoreboard_directory
tree = BeautifulSoup(open(base_scoreboard_directory + base_file_name),
'lxml')
for game in tree.find_all('game'):
game_dir_path = base_scoreboard_directory + 'gid_' + str(
game['id']) + '/'
line_score_url = url + 'gid_' + str(game['id']) + '/linescore.xml'
players_file_url = url + 'gid_' + str(game['id']) + '/players.xml'
game_events_url = url + 'gid_' + str(game['id']) + '/game_events.xml'
box_score_url = url + 'gid_' + str(game['id']) + '/boxscore.xml'
print game_dir_path
# print line_score_url
# print players_file_url
# print game_events_url
print box_score_url
retrieve_data_file(line_score_url, game_dir_path + 'linescore.xml')
retrieve_data_file(players_file_url, game_dir_path + 'players.xml')
retrieve_data_file(game_events_url, game_dir_path + 'game_events.xml')
retrieve_data_file(box_score_url, game_dir_path + 'boxscore.xml')
def download_scoreboard(url, year, month, day):
scoreboard_url = url + 'scoreboard.xml'
testfile = urllib.URLopener()
file_name = scoreboard_url[30:].replace('/', '_')
print file_name
month_as_string = convert_month_int_to_string(month)
directory = str(
str(os.getcwd()) + '/data/' + str(year) + '/' + month_as_string + '/' +
convert_number_to_string(day) + '/')
print scoreboard_url
print directory
testfile.retrieve(scoreboard_url, directory + str(file_name))
def process_player_data_for_today(year, month, day, players_by_id):
base_file_name = get_scoreboard_file(year, month, day)
base_scoreboard_directory = str(str(os.getcwd()) + '/data/' + str(year) + '/' +
convert_month_int_to_string(month) + '/' +
convert_number_to_string(day) + '/')
print base_scoreboard_directory
try:
tree = BeautifulSoup(open(base_scoreboard_directory + base_file_name), 'lxml')
for game in tree.find_all('game'):
game_dir_path = base_scoreboard_directory + 'gid_' + str(game['id']) + '/'
get_players_data(game_dir_path, players_by_id, base_scoreboard_directory)
except Exception as exception:
print exception
def process_player_data_for_today(year, month, day, all_stats_array):
base_file_name = get_scoreboard_file(year, month, day)
base_scoreboard_directory = str(
str(os.getcwd()) + '/data/' + str(year) + '/' +
convert_month_int_to_string(month) + '/' +
convert_number_to_string(day) + '/')
print base_scoreboard_directory
tree = BeautifulSoup(open(base_scoreboard_directory + base_file_name),
'lxml')
for game in tree.find_all('game'):
game_dir_path = base_scoreboard_directory + 'gid_' + str(
game['id']) + '/'
get_starter_stats(game_dir_path, all_stats_array)
def create_month_of_data_directories(month, start_day, end_day, all_star_day):
start_and_end = get_default_start_and_end_of_month(month)
if not start_day:
start_day = start_and_end['start']
if not end_day:
end_day = start_and_end['end']
month_as_string = convert_month_int_to_string(month)
os.chdir(month_as_string)
print os.getcwd()
for day in range(start_day, end_day + 1):
# skip the all star break
if month == 7 and day in range(all_star_day - 1, all_star_day + 3):
continue
day_dir_path = str(os.getcwd()) + '/' + convert_number_to_string(day)
if not os.path.exists(day_dir_path):
os.makedirs(day_dir_path)
os.chdir('..')
def process_player_data_for_today(year, month, day, all_events_array):
base_file_name = get_scoreboard_file(year, month, day)
base_scoreboard_directory = str(
str(os.getcwd()) + '/data/' + str(year) + '/' +
convert_month_int_to_string(month) + '/' +
convert_number_to_string(day) + '/')
print base_scoreboard_directory
tree = BeautifulSoup(open(base_scoreboard_directory + base_file_name),
'lxml')
for game in tree.find_all('game'):
game_dir_path = base_scoreboard_directory + 'gid_' + str(
game['id']) + '/'
team_name_abbreviations = get_linescore_data(game_dir_path)
expected_player_stats = get_boxscore_data(game_dir_path)
outcome = get_outcomes_data(game_dir_path, expected_player_stats,
team_name_abbreviations,
base_scoreboard_directory)
if outcome:
all_events_array[game_dir_path] = outcome
def create_month_of_game_directories(year, month, start_day, end_day, all_star_day):
start_and_end = get_default_start_and_end_of_month(month)
if not start_day:
start_day = start_and_end['start']
if not end_day:
end_day = start_and_end['end']
month_as_string = convert_month_int_to_string(month)
os.chdir(month_as_string)
# print os.getcwd()
for day in range(start_day, end_day + 1):
if month == 7 and day in range(all_star_day - 1, all_star_day + 3):
continue # skip the all star break
os.chdir(convert_number_to_string(day))
# print os.getcwd()
create_folders_from_xml_scoreboard(year, month, day)
os.chdir('..')
os.chdir('..')