def ownership(self, day=None, all_missing=False):
'''
Args:
day(str): in mm_dd_YYYY format
all_missing(bool): single day or all missing days in season?
Returns:
players(list): of player ownership dict
'''
if day:
day = convert_format(day, 'fl')
own = self.scraper.ownership(day)
if self.insert_db:
self.db.insert_ownership(own, convert_format(day, 'std'))
return own
elif all_missing:
owns = {}
for day in self.db.select_list(
'SELECT game_date FROM missing_ownership'):
daystr = datetostr(day, 'fl')
own = self.scraper.ownership(daystr)
self.db.insert_ownership(own, convert_format(daystr, 'std'))
owns[daystr] = own
return owns
else:
raise ValueError('must provide day or set all_missing to True')
def salaries(self, day=None, all_missing=False):
'''
Args:
day(str): in mm_dd_YYYY format
Returns:
players(list): of player dict
'''
if day:
sals = self.parser.dk_salaries(self.scraper.model(day), day)
if self.insert_db and sals:
self.db.insert_salaries(sals, game_date=convert_format(day, 'std'))
return sals
elif all_missing:
salaries = {}
for day in self.db.select_list(missing_salaries()):
daystr = datetostr(day, 'fl')
sals = self.parser.dk_salaries(self.scraper.model(daystr), daystr)
salaries[datetostr(day, 'nba')] = sals
logging.debug('got salaries for {}'.format(daystr))
time.sleep(1)
if self.insert_db and salaries:
self.db.insert_salaries_dict(salaries)
return salaries
else:
raise ValueError('must provide day or set all_missing to True')
def odds(self, day=None, all_missing=False):
'''
Args:
day(str): in mm_dd_YYYY format
all_missing(bool): single day or all missing days in season?
Returns:
players(list): of player ownership dict
'''
if day:
try:
day = convert_format(day, 'db')
odds = self.parser.odds(self.scraper.odds(day), day)
self.db.insert_dicts(game_odds(odds), 'odds')
return odds
except:
logging.exception('could not get odds for {}'.format(day))
elif all_missing:
allodds = {}
for day in self.db.select_list(missing_odds()):
try:
odds = self.parser.odds(self.scraper.odds(day), day)
self.db.insert_dicts(game_odds(odds), 'odds')
allodds[day] = odds
except:
logging.exception('could not get odds for {}'.format(day))
return allodds
else:
raise ValueError('must provide day or set all_missing to True')
self.db.execute(update_odds())
def game_odds(odds):
newodds = []
for o in odds:
fixed = {}
fixed['away'] = long_to_code(o.get('away'))
fixed['home'] = long_to_code(o.get('home'))
try:
fixed['consensus_spread'] = float(o.get('consensus_spread'))
fixed['consensus_game_ou'] = float(o.get('consensus_total'))
except:
pass
fixed['opening_spread'] = o.get('opening_spread')
fixed['opening_game_ou'] = o.get('opening_game_ou')
fixed['gamecode'] = '{}/{}{}'.format(o.get('game_date'), fixed['away'],
fixed['home'])
fixed['game_date'] = convert_format(o.get('game_date'), 'nba')
newodds.append(fixed)
for newo in newodds:
t1 = deepcopy(newo)
t1['team_code'] = newo['away']
t1.pop('away', None)
t1.pop('home', None)
t1['opening_spread'] = 0 - t1['opening_spread']
t1['consensus_spread'] = 0 - t1['consensus_spread']
t2 = deepcopy(newo)
t2['team_code'] = newo['home']
t2.pop('away', None)
t2.pop('home', None)
return newodds
def get_wayback(self, url, d=None, max_delta=None):
'''
Gets page from the wayback machine
Args:
url: of the site you want, not the wayback machine
d: datestring, if None then get most recent one
max_delta: int, how many days off can the last page be from the requested date
Returns:
content: HTML string
'''
content = None
if not d:
d = today('db')
else:
d = convert_format(d, 'db')
resp = self.get_json(self.wburl.format(url, d))
if resp:
ts = resp['archived_snapshots']['closest']['timestamp'][:8]
if max_delta:
closest_url = resp['archived_snapshots']['closest']['url']
if subtract_datestr(d, ts) <= max_delta:
content = self.get(
resp['archived_snapshots']['closest']['url'])
else:
logging.error('page is too old: {}'.format(ts))
else:
closest_url = resp['archived_snapshots']['closest']['url']
return self.get(closest_url)
else:
logging.error('url unavailable on wayback machine')
return content, ts
def odds(self, game_date):
'''
Gets odds for specific date
Args:
game_date: str
'''
results = []
q = """SELECT data FROM rotogrinders WHERE game_date = '{}'"""
data = self.select_scalar(q.format(convert_format(game_date, 'nba')))
for gid, gdata in data.items():
hid = gdata['data']['home_id']
vid = gdata['data']['away_id']
hcode = gdata['data']['team_home']['data']['hashtag']
vcode = gdata['data']['team_away']['data']['hashtag']
vegas = gdata['data']['vegas']
# they have odds with various timestamps during the day
if vegas:
stamps = [
datetime.strptime(k, '%Y-%m-%d %H:%M:%S') for k in vegas
]
if stamps:
fmt = '%Y-%m-%d %H:%M:%S'
maxts = datetime.strftime(max(stamps), fmt)
newest = vegas[maxts]
results.append({
'game_id':
gid,
'ts':
maxts,
'visitor_team_id':
vid,
'visitor_team_code':
vcode.upper(),
'visitor_team_spread':
float(newest['spread']['spread_visiting']),
'home_team_id':
hid,
'home_team_code':
hcode.upper(),
'home_team_spread':
float(newest['spread']['spread_home']),
'game_ou':
float(newest['total']['total_points']),
'delta_visiting':
float(newest['team_total']['delta_visiting']),
'delta_home':
float(newest['team_total']['delta_home']),
'team_total_home':
float(newest['team_total']['team_total_home']),
'team_total_visiting':
float(newest['team_total']['team_total_visiting'])
})
return results
def salaries(self, content, game_date, site, ssv='0'):
'''
Returns list of dictionaries with player salary information
Arguments:
content (str): HTML string
game_date (str): likely YYYYmmdd
site (str): name of site ('dk', 'fd', 'yh', etc.)
Returns:
salaries (list): list of dictionaries representing player & his salary on a given date on a given site
'''
sals = []
if ssv == '1':
rows = self._pre_ssv(content)
headers = self._salaries_headers()
wanted = ['game_date', 'site_position', 'site_player_name', 'salary']
for row in rows[1:]:
cells = row.split(';')
sal = {k:v for k,v in dict(list(zip(headers, cells))).items() if k in wanted}
sal['site'] = site
if sal.get('salary', None):
sal['salary'] = re.sub("[^0-9]", "", sal['salary'])
sals.append(sal)
else:
season = int(in_what_season(game_date)[0:4]) + 1
soup = BeautifulSoup(content, 'lxml')
t = soup.find('table', {'cellspacing': 5})
headers = ['dfs_site', 'season', 'game_date', 'source', 'source_player_id', 'source_player_name', 'team_code',
'dfs_position', 'salary']
for tr in t.find_all('tr'):
try:
tds = tr.find_all('td')
a = tr.find('a')
if a and a.get('href', '') and '?' in a.get('href'):
source_player_id = int(a.get('href', '').split('?')[1].replace('x', ''))
dfs_position = tds[0].text
source_player_name = a.text.replace('^', '').replace('*', '')
salary = int(tds[3].text.replace('$', '').replace(',', ''))
team_code = tds[4].text.strip().upper()
vals = ['dk', season, convert_format(game_date, 'nba'), 'rotoguru', source_player_id,
source_player_name, team_code,
dfs_position, salary]
sals.append(dict(zip(headers, vals)))
except Exception as e:
logging.exception(e)
return sals
def team_opponent_dashboards_table(dash, as_of):
'''
Args:
dash:
as_of:
Returns:
'''
topp = []
omit = ['CFID', 'CFPARAMS', 'COMMENT', 'TEAM_NAME']
for team in dash:
fixed_team = {k.lower(): v for k, v in team.items() if k not in omit}
fixed_team.pop('team_name', None)
fixed_team['as_of'] = convert_format(as_of, 'nba')
topp.append(fixed_team)
return topp
