def download_team(data_dir, team, year=time.strftime('%Y', time.localtime())): data_dir = os.path.join(data_dir, "teams", str(year)) mkdir_p(data_dir) filename = "{team}.html".format(team=team) filename = os.path.join(data_dir, filename) fp = open(filename, "wb") curl = pycurl.Curl() curl.setopt(pycurl.URL, base_team_url.format(team=team, year=year)) curl.setopt(pycurl.FOLLOWLOCATION, 1) curl.setopt(pycurl.MAXREDIRS, 5) curl.setopt(pycurl.CONNECTTIMEOUT, 30) curl.setopt(pycurl.TIMEOUT, 300) curl.setopt(pycurl.NOSIGNAL, 1) curl.setopt(pycurl.WRITEDATA, fp) try: curl.perform() except pycurl.error: import traceback traceback.print_exc(file=sys.stderr) curl.close() fp.close() return
def download_draft(data_dir, year): data_dir = os.path.join(data_dir, "draft", str(year)) mkdir_p(data_dir) filename = "draft.html" filename = os.path.join(data_dir, filename) fp = open(filename, "wb") curl = pycurl.Curl() curl.setopt(pycurl.URL, base_draft_url.format(year=year)) curl.setopt(pycurl.FOLLOWLOCATION, 1) curl.setopt(pycurl.MAXREDIRS, 5) curl.setopt(pycurl.CONNECTTIMEOUT, 30) curl.setopt(pycurl.TIMEOUT, 300) curl.setopt(pycurl.NOSIGNAL, 1) curl.setopt(pycurl.WRITEDATA, fp) try: curl.perform() except pycurl.error: import traceback traceback.print_exc(file=sys.stderr) curl.close() fp.close() return
def write_pkl(self): """ """ dst_dir = os.path.join(self.data_dir, "processed_data", str(self.year)) mkdir_p(dst_dir) dst_file = os.path.join(dst_dir, "league_data.pkl") self.df.to_pickle(dst_file) dst_file = os.path.join(dst_dir, "league_player_data.pkl") self.team_df.to_pickle(dst_file)
def write_pkl(self): """ """ dst_dir = os.path.join(self.data_dir, 'processed_data', str(self.year)) mkdir_p(dst_dir) dst_file = os.path.join(dst_dir, 'league_data.pkl') self.df.to_pickle(dst_file) dst_file = os.path.join(dst_dir, 'league_player_data.pkl') self.team_df.to_pickle(dst_file)
def __init__(self, data_dir, year): default_processed_data_dir = os.path.join(data_dir, 'processed_data') default_plot_dir = os.path.join(data_dir, 'plots') self.save_dir = os.path.join(default_plot_dir, year) fname = os.path.join(default_processed_data_dir, year, 'team_data.pkl') self.df = pd.read_pickle(fname) self.year = year self.make_positional_df() mkdir_p(self.save_dir)
def download_league(data_dir, leagueID, year): """ fetch standings and team data from espn.com """ espn_pages = [] league_dir = os.path.join(data_dir, 'league', str(year)) mkdir_p(league_dir) leagueURL = "http://games.espn.go.com/fba/leaguerosters?" +\ "leagueId={0}&seasonId={1}".format(leagueID, year) league_filename = "league.html" league_filename = os.path.join(league_dir, league_filename) espn_pages.append({'url': leagueURL, 'filename': league_filename}) standingsURL = "http://games.espn.go.com/fba/standings?" +\ "leagueId={0}&seasonId={1}".format(leagueID, year) standings_filename = "standings.html" standings_filename = os.path.join(league_dir, standings_filename) espn_pages.append({'url': standingsURL, 'filename': standings_filename}) draft_recap_url = 'http://games.espn.go.com/fba/tools/draftrecap?' +\ 'leagueId={leagueID}&seasonId={year}' draft_recap_url = draft_recap_url.format(leagueID=leagueID, year=year) draft_recap_filename = "draft_recap.html" draft_recap_filename = os.path.join(league_dir, draft_recap_filename) espn_pages.append({'url': draft_recap_url, 'filename': draft_recap_filename}) schedule_url = 'http://games.espn.go.com/fba/schedule?' +\ 'leagueId={leagueID}&seasonId={year}' schedule_url = schedule_url.format(year=year, leagueID=leagueID) schedule_filename = "schedule.html" schedule_filename = os.path.join(league_dir, schedule_filename) espn_pages.append({'url': schedule_url, 'filename': schedule_filename}) for page in espn_pages: fd = open(page['filename'], "w") c = pycurl.Curl() c.setopt(c.URL, page['url']) c.setopt(c.WRITEDATA, fd) try: c.perform() except pycurl.error: import traceback traceback.print_exc(file=sys.stderr) fd.close() c.close()
def write_html(self): """ This needs a re-write """ for p in self.pages: dump_html_dir = os.path.join(self.html_dir, str(p['year'])) mkdir_p(dump_html_dir) with open(os.path.join(dump_html_dir, p['href']), 'w') as fd: text = self.render_tables(p) fd.write(text.encode('UTF-8')) self.render_plots() self.render_index()
def get_player_stats(data_dir, year): d = os.path.join(data_dir, 'raw_data', 'teams', str(year)) pkl = os.path.join(data_dir, 'processed_data', str(year)) draft_dir = os.path.join(data_dir, 'raw_data', 'draft') df = get_players(d, year) if os.path.isdir(draft_dir): draft_df = get_draft(draft_dir) df = augment_draft_data(df, draft_df) df = augment_fantasy_teams(df, data_dir) df = augment_minutes(df) df = augment_value(df) df = augment_price(df) mkdir_p(pkl) pkl = os.path.join(pkl, 'team_data.pkl') df.to_pickle(pkl)
def download_draft(data_dir, year): data_dir = os.path.join(data_dir, "draft", str(year)) mkdir_p(data_dir) filename = "draft.html" filename = os.path.join(data_dir, filename) url = base_draft_url.format(year=year) with open(filename, "wb") as fd: r = requests.get(url) if r.status_code == 200: fd.write(r.text) else: print 'Download {0} failed'.format(url) return
def render_plots(self): ''' ''' years = [ii['year'] for ii in self.plot_data] data = self.plot_data for year in years: plots_dir = os.path.join(self.html_dir, str(year)) mkdir_p(plots_dir) # Filter all plots by year plot_files = [ii for ii in data if ii['year'] == year] with open(os.path.join(plots_dir, 'plots.html'), 'w') as fd: text = self.chartsTemplate.render(title='Plots', years=self.years, plots=plot_files) fd.write(text.encode('UTF-8'))
def download_team(data_dir, team, year=time.strftime('%Y', time.localtime())): data_dir = os.path.join(data_dir, "teams", str(year)) mkdir_p(data_dir) filename = "{team}.html".format(team=team) filename = os.path.join(data_dir, filename) url = base_team_url.format(team=team, year=year) with open(filename, 'wb') as fd: r = requests.get(url) if r.status_code == 200: fd.write(r.text) else: print 'Downloading {0} failed'.format(url) return return
def __init__(self, data_dir): self.data_dir = data_dir self.processed_dir = os.path.join(self.data_dir, "processed_data") self.plots_dir = os.path.join(self.data_dir, 'plots') if not os.path.isdir(self.data_dir): raise IOError("{0} not a valid directory".format(self.data_dir)) self.dataframes = [] self.pages = [] self.html_dir = os.path.join(self.data_dir, 'html') mkdir_p(self.html_dir) j2_env = Environment(loader=FileSystemLoader('templates'), trim_blocks=True) self.baseTemplate = j2_env.get_template('fantasy-template.html') self.tocTemplate = j2_env.get_template('toc.html') self.posTemplate = j2_env.get_template('positional-template.html') self.chartsTemplate = j2_env.get_template('charts-template.html') self.copy_static_files()