def write_draft_experts(self,
year=utils.get_current_season(),
update_master=True):
"""
Write the experts currently available for draft rankings to CSV files. One CSV file per expert/position/scoring
combination. For example:
./experts/2019/draft/2019_expert_list_draft_OVERALL_STD.csv
Repeatedly calls the get_draft_experts method and writes the returned dataframe to CSV.
Maintains a "master" file containing information about every expert that has ever been scraped.
NOTE: check marks will probably not be accurate if this method is run for a year earlier than the current year.
:param year: the season year to return experts for (int) (defaults to current year)
:param update_master: if True, update the master file with the results
"""
combined_df = pd.DataFrame()
directory = os.path.join(self.expert_dir, str(year), 'draft')
# first do the overall rankings
for scoring in ['STD', 'HALF', 'PPR']:
df = self.get_draft_experts(pos='ALL', scoring=scoring, year=year)
file = os.path.join(
directory,
f'{year}_expert_list_draft_OVERALL_{scoring.upper()}.csv')
utils.write_df(df=df, filepath=file, verbose=self.verbose)
if update_master:
combined_df = combined_df.append(df)
# then do positional rankings
for pos in _draft_positions:
for scoring in _pos_scoring_dict[pos]:
df = self.get_draft_experts(pos=pos,
scoring=scoring,
year=year)
file = os.path.join(
directory,
f'{year}_expert_list_draft_{pos.upper()}_{scoring.upper()}.csv'
)
utils.write_df(df=df, filepath=file, verbose=self.verbose)
if update_master:
combined_df = combined_df.append(df)
if update_master:
rank_id = f'{year}_draft'
combined_df = (combined_df.groupby(
['expert_id', 'expert_name', 'site'],
as_index=False).agg({'timestamp': 'min'}))
self._update_master(df=combined_df, rank_id=rank_id)
def write_draft_rankings(self,
year=utils.get_current_season(),
start_exp=0):
"""
Write the draft rankings for all available FantasyPros experts to CSV files. Available experts are read from
the expert master file. One CSV file per expert/position/scoring combination. For example:
./rankings/2019/draft/2019_draft_9_Scott Pianowski_Yahoo! Sports_OVERALL_STD.csv
Repeatedly calls the get_draft_rankings method and writes the returned dataframe to CSV.
:param year: the season year to return rankings for (int) (defaults to current year)
:param start_exp: the expert ID number to start with (defaults to 0 to scrape all experts)
"""
expert_df = self._get_all_experts()
for i, row in expert_df.iterrows():
expert_id = row['expert_id']
expert_name = row['expert_name']
site = row['site']
if expert_id < start_exp:
continue
# first get overall rankings
for scoring in ['STD', 'HALF', 'PPR']:
rank_df = self.get_draft_rankings(expert_id=expert_id,
scoring=scoring,
pos='ALL',
year=year)
# write to CSV if results were found
if rank_df.shape[0] > 0:
filename = f'{year}_draft_{expert_id}_{expert_name}_{site}_OVERALL_{scoring}.csv'
path = os.path.join(self.ranking_dir, str(year), 'draft',
filename)
utils.write_df(rank_df, path, verbose=self.verbose)
# then get positional rankings
for pos in _draft_positions:
for scoring in _pos_scoring_dict[pos]:
rank_df = self.get_draft_rankings(expert_id=expert_id,
scoring=scoring,
pos=pos,
year=year)
# write to CSV if results were found
if rank_df.shape[0] > 0:
filename = f'{year}_draft_{expert_id}_{expert_name}_{site}_{pos.upper()}_{scoring}.csv'
path = os.path.join(self.ranking_dir, str(year),
'draft', filename)
utils.write_df(rank_df, path, verbose=self.verbose)
def write_weekly_experts(self,
year=utils.get_current_season(),
week=utils.get_current_week(),
update_master=True):
"""
Write the experts currently available for weekly rankings to CSV files. One CSV file per expert/position/scoring
combination. For example:
./experts/2019/weekly/week1/2019_expert_list_week1_RB_STD.csv
Repeatedly calls the get_weekly_experts method and writes the returned dataframe to CSV.
Maintains a "master" file containing information about every expert that has ever been scraped.
NOTE: check marks will probably not be accurate if this method is run for a year earlier than the current year.
:param year: the season year to return experts for (int) (defaults to current year)
:param week: the week number to return experts for (int) (defaults to current week number)
:param update_master: if True, update the master file with the results
"""
if week == 0:
print(f"Writing draft experts because 'week' was set to 0.")
# this won't return anything, but it will prevent the rest of this method from running
return self.write_draft_experts(year=year,
update_master=update_master)
combined_df = pd.DataFrame()
directory = os.path.join(self.expert_dir, str(year), 'weekly',
f'week{week}')
for pos in _pos_scoring_dict.keys():
for scoring in _pos_scoring_dict[pos]:
df = self.get_weekly_experts(pos=pos,
scoring=scoring,
year=year,
week=week)
file = os.path.join(
directory,
f'{year}_expert_list_week{week}_{pos.upper()}_{scoring.upper()}.csv'
)
utils.write_df(df=df, filepath=file, verbose=self.verbose)
if update_master:
combined_df = combined_df.append(df)
if update_master:
rank_id = f'{year}_week{week}'
combined_df = (combined_df.groupby(
['expert_id', 'expert_name', 'site'],
as_index=False).agg({'timestamp': 'min'}))
self._update_master(df=combined_df, rank_id=rank_id)
def write_weekly_rankings(self,
year=utils.get_current_season(),
week=utils.get_current_week(),
start_exp=0):
"""
Write the draft rankings for all available FantasyPros experts to CSV files. Available experts are read from
the expert master file. One CSV file per expert/position/scoring combination. For example:
./rankings/2019/weekly/2019_week1_9_Scott Pianowski_Yahoo! Sports_RB_STD.csv
Repeatedly calls the get_weekly_rankings method and writes the returned dataframe to CSV.
:param year: the season year to return rankings for (int) (defaults to current year)
:param week: the week number to return rankings for (int) (defaults to current week number)
:param start_exp: the expert ID number to start with (defaults to 0 to scrape all experts)
"""
if week == 0:
print(f"Writing draft rankings because 'week' was set to 0.")
# this won't return anything, but returning the value will prevent the rest of this method from running
return self.write_draft_rankings(year=year, start_exp=start_exp)
expert_df = self._get_all_experts()
for i, row in expert_df.iterrows():
expert_id = row['expert_id']
expert_name = row['expert_name']
site = row['site']
if expert_id < start_exp:
continue
for pos in _pos_scoring_dict.keys():
for scoring in _pos_scoring_dict[pos]:
rank_df = self.get_weekly_rankings(expert_id=expert_id,
scoring=scoring,
pos=pos,
year=year,
week=week)
# write to CSV if results were found
if rank_df.shape[0] > 0:
filename = f'{year}_week{week}_{expert_id}_{expert_name}_{site}_{pos}_{scoring}.csv'
path = os.path.join(self.ranking_dir, str(year),
'weekly', f'week{week}', filename)
utils.write_df(rank_df, path, verbose=self.verbose)
def _update_master(self, df, rank_id):
"""
Update the master file with a list of available experts. Keep track of expert ID/name/site mappings and
record the first and latest appearance of each expert.
NOTE: the name of the master file is specified at the top of this script.
:param df: dataframe of experts
:param rank_id: string to identify when this expert produced a ranking (e.g., '2019_draft')
"""
try:
existing_df = pd.read_csv(self.master_file, dtype=str)
except FileNotFoundError:
if self.verbose:
print(f'File not found. Creating {self.master_file}')
existing_df = pd.DataFrame(
{
'expert_id': [],
'expert_name': [],
'site': [],
'first_timestamp': [],
'latest_timestamp': [],
'first_appearance': [],
'latest_appearance': []
},
dtype=str)
for i, row in df.iterrows():
conds = (existing_df['expert_id'] == row['expert_id']) & \
(existing_df['expert_name'] == row['expert_name']) & \
(existing_df['site'] == row['site'])
existing_row = existing_df.loc[conds]
# if there's no row with this expert, ID, site combo, add a new row
if existing_row.shape[0] == 0:
existing_df = existing_df.append(
pd.DataFrame({
'expert_id': [row['expert_id']],
'expert_name': [row['expert_name']],
'site': [row['site']],
'first_timestamp': [row['timestamp']],
'latest_timestamp': [row['timestamp']],
'first_appearance': [rank_id],
'latest_appearance': [rank_id]
}))
# otherwise update the existing row
else:
existing_df.loc[conds & (
existing_df['first_timestamp'] > row['timestamp']),
'first_timestamp'] = row['timestamp']
existing_df.loc[conds & (
existing_df['latest_timestamp'] < row['timestamp']),
'latest_timestamp'] = row['timestamp']
existing_df.loc[conds &
(existing_df['first_appearance'] > rank_id),
'first_appearance'] = rank_id
existing_df.loc[conds &
(existing_df['latest_appearance'] < rank_id),
'latest_appearance'] = rank_id
# drop duplicates before writing in case we get duplicate rows (shouldn't happen)
existing_df['expert_id'] = existing_df['expert_id'].astype(int)
existing_df = existing_df.drop_duplicates().sort_values('expert_id')
utils.write_df(df=existing_df,
filepath=self.master_file,
verbose=self.verbose)