def download_team(data_dir, team, year=time.strftime('%Y', time.localtime())):
data_dir = os.path.join(data_dir, "teams", str(year))
mkdir_p(data_dir)
filename = "{team}.html".format(team=team)
filename = os.path.join(data_dir, filename)
fp = open(filename, "wb")
curl = pycurl.Curl()
curl.setopt(pycurl.URL, base_team_url.format(team=team, year=year))
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.CONNECTTIMEOUT, 30)
curl.setopt(pycurl.TIMEOUT, 300)
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.WRITEDATA, fp)
try:
curl.perform()
except pycurl.error:
import traceback
traceback.print_exc(file=sys.stderr)
curl.close()
fp.close()
return
def download_draft(data_dir, year):
data_dir = os.path.join(data_dir, "draft", str(year))
mkdir_p(data_dir)
filename = "draft.html"
filename = os.path.join(data_dir, filename)
fp = open(filename, "wb")
curl = pycurl.Curl()
curl.setopt(pycurl.URL, base_draft_url.format(year=year))
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.CONNECTTIMEOUT, 30)
curl.setopt(pycurl.TIMEOUT, 300)
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.WRITEDATA, fp)
try:
curl.perform()
except pycurl.error:
import traceback
traceback.print_exc(file=sys.stderr)
curl.close()
fp.close()
return
def write_pkl(self):
"""
"""
dst_dir = os.path.join(self.data_dir, "processed_data", str(self.year))
mkdir_p(dst_dir)
dst_file = os.path.join(dst_dir, "league_data.pkl")
self.df.to_pickle(dst_file)
dst_file = os.path.join(dst_dir, "league_player_data.pkl")
self.team_df.to_pickle(dst_file)
def write_pkl(self):
"""
"""
dst_dir = os.path.join(self.data_dir, 'processed_data', str(self.year))
mkdir_p(dst_dir)
dst_file = os.path.join(dst_dir, 'league_data.pkl')
self.df.to_pickle(dst_file)
dst_file = os.path.join(dst_dir, 'league_player_data.pkl')
self.team_df.to_pickle(dst_file)
def __init__(self, data_dir, year):
default_processed_data_dir = os.path.join(data_dir, 'processed_data')
default_plot_dir = os.path.join(data_dir, 'plots')
self.save_dir = os.path.join(default_plot_dir, year)
fname = os.path.join(default_processed_data_dir, year, 'team_data.pkl')
self.df = pd.read_pickle(fname)
self.year = year
self.make_positional_df()
mkdir_p(self.save_dir)
def __init__(self, data_dir, year):
default_processed_data_dir = os.path.join(data_dir, 'processed_data')
default_plot_dir = os.path.join(data_dir, 'plots')
self.save_dir = os.path.join(default_plot_dir, year)
fname = os.path.join(default_processed_data_dir, year, 'team_data.pkl')
self.df = pd.read_pickle(fname)
self.year = year
self.make_positional_df()
mkdir_p(self.save_dir)
def download_league(data_dir, leagueID, year):
"""
fetch standings and team data from espn.com
"""
espn_pages = []
league_dir = os.path.join(data_dir, 'league', str(year))
mkdir_p(league_dir)
leagueURL = "http://games.espn.go.com/fba/leaguerosters?" +\
"leagueId={0}&seasonId={1}".format(leagueID, year)
league_filename = "league.html"
league_filename = os.path.join(league_dir, league_filename)
espn_pages.append({'url': leagueURL, 'filename': league_filename})
standingsURL = "http://games.espn.go.com/fba/standings?" +\
"leagueId={0}&seasonId={1}".format(leagueID, year)
standings_filename = "standings.html"
standings_filename = os.path.join(league_dir, standings_filename)
espn_pages.append({'url': standingsURL, 'filename': standings_filename})
draft_recap_url = 'http://games.espn.go.com/fba/tools/draftrecap?' +\
'leagueId={leagueID}&seasonId={year}'
draft_recap_url = draft_recap_url.format(leagueID=leagueID, year=year)
draft_recap_filename = "draft_recap.html"
draft_recap_filename = os.path.join(league_dir, draft_recap_filename)
espn_pages.append({'url': draft_recap_url,
'filename': draft_recap_filename})
schedule_url = 'http://games.espn.go.com/fba/schedule?' +\
'leagueId={leagueID}&seasonId={year}'
schedule_url = schedule_url.format(year=year, leagueID=leagueID)
schedule_filename = "schedule.html"
schedule_filename = os.path.join(league_dir, schedule_filename)
espn_pages.append({'url': schedule_url, 'filename': schedule_filename})
for page in espn_pages:
fd = open(page['filename'], "w")
c = pycurl.Curl()
c.setopt(c.URL, page['url'])
c.setopt(c.WRITEDATA, fd)
try:
c.perform()
except pycurl.error:
import traceback
traceback.print_exc(file=sys.stderr)
fd.close()
c.close()
def download_league(data_dir, leagueID, year):
"""
fetch standings and team data from espn.com
"""
espn_pages = []
league_dir = os.path.join(data_dir, 'league', str(year))
mkdir_p(league_dir)
leagueURL = "http://games.espn.go.com/fba/leaguerosters?" +\
"leagueId={0}&seasonId={1}".format(leagueID, year)
league_filename = "league.html"
league_filename = os.path.join(league_dir, league_filename)
espn_pages.append({'url': leagueURL, 'filename': league_filename})
standingsURL = "http://games.espn.go.com/fba/standings?" +\
"leagueId={0}&seasonId={1}".format(leagueID, year)
standings_filename = "standings.html"
standings_filename = os.path.join(league_dir, standings_filename)
espn_pages.append({'url': standingsURL, 'filename': standings_filename})
draft_recap_url = 'http://games.espn.go.com/fba/tools/draftrecap?' +\
'leagueId={leagueID}&seasonId={year}'
draft_recap_url = draft_recap_url.format(leagueID=leagueID, year=year)
draft_recap_filename = "draft_recap.html"
draft_recap_filename = os.path.join(league_dir, draft_recap_filename)
espn_pages.append({'url': draft_recap_url,
'filename': draft_recap_filename})
schedule_url = 'http://games.espn.go.com/fba/schedule?' +\
'leagueId={leagueID}&seasonId={year}'
schedule_url = schedule_url.format(year=year, leagueID=leagueID)
schedule_filename = "schedule.html"
schedule_filename = os.path.join(league_dir, schedule_filename)
espn_pages.append({'url': schedule_url, 'filename': schedule_filename})
for page in espn_pages:
fd = open(page['filename'], "w")
c = pycurl.Curl()
c.setopt(c.URL, page['url'])
c.setopt(c.WRITEDATA, fd)
try:
c.perform()
except pycurl.error:
import traceback
traceback.print_exc(file=sys.stderr)
fd.close()
c.close()
def write_html(self):
"""
This needs a re-write
"""
for p in self.pages:
dump_html_dir = os.path.join(self.html_dir, str(p['year']))
mkdir_p(dump_html_dir)
with open(os.path.join(dump_html_dir, p['href']), 'w') as fd:
text = self.render_tables(p)
fd.write(text.encode('UTF-8'))
self.render_plots()
self.render_index()
def get_player_stats(data_dir, year):
d = os.path.join(data_dir, 'raw_data', 'teams', str(year))
pkl = os.path.join(data_dir, 'processed_data', str(year))
draft_dir = os.path.join(data_dir, 'raw_data', 'draft')
df = get_players(d, year)
if os.path.isdir(draft_dir):
draft_df = get_draft(draft_dir)
df = augment_draft_data(df, draft_df)
df = augment_fantasy_teams(df, data_dir)
df = augment_minutes(df)
df = augment_value(df)
df = augment_price(df)
mkdir_p(pkl)
pkl = os.path.join(pkl, 'team_data.pkl')
df.to_pickle(pkl)
def download_draft(data_dir, year):
data_dir = os.path.join(data_dir, "draft", str(year))
mkdir_p(data_dir)
filename = "draft.html"
filename = os.path.join(data_dir, filename)
url = base_draft_url.format(year=year)
with open(filename, "wb") as fd:
r = requests.get(url)
if r.status_code == 200:
fd.write(r.text)
else:
print 'Download {0} failed'.format(url)
return
def render_plots(self):
'''
'''
years = [ii['year'] for ii in self.plot_data]
data = self.plot_data
for year in years:
plots_dir = os.path.join(self.html_dir, str(year))
mkdir_p(plots_dir)
# Filter all plots by year
plot_files = [ii for ii in data if ii['year'] == year]
with open(os.path.join(plots_dir, 'plots.html'), 'w') as fd:
text = self.chartsTemplate.render(title='Plots',
years=self.years,
plots=plot_files)
fd.write(text.encode('UTF-8'))
def download_team(data_dir, team, year=time.strftime('%Y', time.localtime())):
data_dir = os.path.join(data_dir, "teams", str(year))
mkdir_p(data_dir)
filename = "{team}.html".format(team=team)
filename = os.path.join(data_dir, filename)
url = base_team_url.format(team=team, year=year)
with open(filename, 'wb') as fd:
r = requests.get(url)
if r.status_code == 200:
fd.write(r.text)
else:
print 'Downloading {0} failed'.format(url)
return
return
def __init__(self, data_dir):
self.data_dir = data_dir
self.processed_dir = os.path.join(self.data_dir, "processed_data")
self.plots_dir = os.path.join(self.data_dir, 'plots')
if not os.path.isdir(self.data_dir):
raise IOError("{0} not a valid directory".format(self.data_dir))
self.dataframes = []
self.pages = []
self.html_dir = os.path.join(self.data_dir, 'html')
mkdir_p(self.html_dir)
j2_env = Environment(loader=FileSystemLoader('templates'),
trim_blocks=True)
self.baseTemplate = j2_env.get_template('fantasy-template.html')
self.tocTemplate = j2_env.get_template('toc.html')
self.posTemplate = j2_env.get_template('positional-template.html')
self.chartsTemplate = j2_env.get_template('charts-template.html')
self.copy_static_files()
