def ownership(self, day=None, all_missing=False): ''' Args: day(str): in mm_dd_YYYY format all_missing(bool): single day or all missing days in season? Returns: players(list): of player ownership dict ''' if day: day = convert_format(day, 'fl') own = self.scraper.ownership(day) if self.insert_db: self.db.insert_ownership(own, convert_format(day, 'std')) return own elif all_missing: owns = {} for day in self.db.select_list( 'SELECT game_date FROM missing_ownership'): daystr = datetostr(day, 'fl') own = self.scraper.ownership(daystr) self.db.insert_ownership(own, convert_format(daystr, 'std')) owns[daystr] = own return owns else: raise ValueError('must provide day or set all_missing to True')
def salaries(self, day=None, all_missing=False): ''' Args: day(str): in mm_dd_YYYY format Returns: players(list): of player dict ''' if day: sals = self.parser.dk_salaries(self.scraper.model(day), day) if self.insert_db and sals: self.db.insert_salaries(sals, game_date=convert_format(day, 'std')) return sals elif all_missing: salaries = {} for day in self.db.select_list(missing_salaries()): daystr = datetostr(day, 'fl') sals = self.parser.dk_salaries(self.scraper.model(daystr), daystr) salaries[datetostr(day, 'nba')] = sals logging.debug('got salaries for {}'.format(daystr)) time.sleep(1) if self.insert_db and salaries: self.db.insert_salaries_dict(salaries) return salaries else: raise ValueError('must provide day or set all_missing to True')
def odds(self, day=None, all_missing=False): ''' Args: day(str): in mm_dd_YYYY format all_missing(bool): single day or all missing days in season? Returns: players(list): of player ownership dict ''' if day: try: day = convert_format(day, 'db') odds = self.parser.odds(self.scraper.odds(day), day) self.db.insert_dicts(game_odds(odds), 'odds') return odds except: logging.exception('could not get odds for {}'.format(day)) elif all_missing: allodds = {} for day in self.db.select_list(missing_odds()): try: odds = self.parser.odds(self.scraper.odds(day), day) self.db.insert_dicts(game_odds(odds), 'odds') allodds[day] = odds except: logging.exception('could not get odds for {}'.format(day)) return allodds else: raise ValueError('must provide day or set all_missing to True') self.db.execute(update_odds())
def game_odds(odds): newodds = [] for o in odds: fixed = {} fixed['away'] = long_to_code(o.get('away')) fixed['home'] = long_to_code(o.get('home')) try: fixed['consensus_spread'] = float(o.get('consensus_spread')) fixed['consensus_game_ou'] = float(o.get('consensus_total')) except: pass fixed['opening_spread'] = o.get('opening_spread') fixed['opening_game_ou'] = o.get('opening_game_ou') fixed['gamecode'] = '{}/{}{}'.format(o.get('game_date'), fixed['away'], fixed['home']) fixed['game_date'] = convert_format(o.get('game_date'), 'nba') newodds.append(fixed) for newo in newodds: t1 = deepcopy(newo) t1['team_code'] = newo['away'] t1.pop('away', None) t1.pop('home', None) t1['opening_spread'] = 0 - t1['opening_spread'] t1['consensus_spread'] = 0 - t1['consensus_spread'] t2 = deepcopy(newo) t2['team_code'] = newo['home'] t2.pop('away', None) t2.pop('home', None) return newodds
def get_wayback(self, url, d=None, max_delta=None): ''' Gets page from the wayback machine Args: url: of the site you want, not the wayback machine d: datestring, if None then get most recent one max_delta: int, how many days off can the last page be from the requested date Returns: content: HTML string ''' content = None if not d: d = today('db') else: d = convert_format(d, 'db') resp = self.get_json(self.wburl.format(url, d)) if resp: ts = resp['archived_snapshots']['closest']['timestamp'][:8] if max_delta: closest_url = resp['archived_snapshots']['closest']['url'] if subtract_datestr(d, ts) <= max_delta: content = self.get( resp['archived_snapshots']['closest']['url']) else: logging.error('page is too old: {}'.format(ts)) else: closest_url = resp['archived_snapshots']['closest']['url'] return self.get(closest_url) else: logging.error('url unavailable on wayback machine') return content, ts
def odds(self, game_date): ''' Gets odds for specific date Args: game_date: str ''' results = [] q = """SELECT data FROM rotogrinders WHERE game_date = '{}'""" data = self.select_scalar(q.format(convert_format(game_date, 'nba'))) for gid, gdata in data.items(): hid = gdata['data']['home_id'] vid = gdata['data']['away_id'] hcode = gdata['data']['team_home']['data']['hashtag'] vcode = gdata['data']['team_away']['data']['hashtag'] vegas = gdata['data']['vegas'] # they have odds with various timestamps during the day if vegas: stamps = [ datetime.strptime(k, '%Y-%m-%d %H:%M:%S') for k in vegas ] if stamps: fmt = '%Y-%m-%d %H:%M:%S' maxts = datetime.strftime(max(stamps), fmt) newest = vegas[maxts] results.append({ 'game_id': gid, 'ts': maxts, 'visitor_team_id': vid, 'visitor_team_code': vcode.upper(), 'visitor_team_spread': float(newest['spread']['spread_visiting']), 'home_team_id': hid, 'home_team_code': hcode.upper(), 'home_team_spread': float(newest['spread']['spread_home']), 'game_ou': float(newest['total']['total_points']), 'delta_visiting': float(newest['team_total']['delta_visiting']), 'delta_home': float(newest['team_total']['delta_home']), 'team_total_home': float(newest['team_total']['team_total_home']), 'team_total_visiting': float(newest['team_total']['team_total_visiting']) }) return results
def salaries(self, content, game_date, site, ssv='0'): ''' Returns list of dictionaries with player salary information Arguments: content (str): HTML string game_date (str): likely YYYYmmdd site (str): name of site ('dk', 'fd', 'yh', etc.) Returns: salaries (list): list of dictionaries representing player & his salary on a given date on a given site ''' sals = [] if ssv == '1': rows = self._pre_ssv(content) headers = self._salaries_headers() wanted = ['game_date', 'site_position', 'site_player_name', 'salary'] for row in rows[1:]: cells = row.split(';') sal = {k:v for k,v in dict(list(zip(headers, cells))).items() if k in wanted} sal['site'] = site if sal.get('salary', None): sal['salary'] = re.sub("[^0-9]", "", sal['salary']) sals.append(sal) else: season = int(in_what_season(game_date)[0:4]) + 1 soup = BeautifulSoup(content, 'lxml') t = soup.find('table', {'cellspacing': 5}) headers = ['dfs_site', 'season', 'game_date', 'source', 'source_player_id', 'source_player_name', 'team_code', 'dfs_position', 'salary'] for tr in t.find_all('tr'): try: tds = tr.find_all('td') a = tr.find('a') if a and a.get('href', '') and '?' in a.get('href'): source_player_id = int(a.get('href', '').split('?')[1].replace('x', '')) dfs_position = tds[0].text source_player_name = a.text.replace('^', '').replace('*', '') salary = int(tds[3].text.replace('$', '').replace(',', '')) team_code = tds[4].text.strip().upper() vals = ['dk', season, convert_format(game_date, 'nba'), 'rotoguru', source_player_id, source_player_name, team_code, dfs_position, salary] sals.append(dict(zip(headers, vals))) except Exception as e: logging.exception(e) return sals
def team_opponent_dashboards_table(dash, as_of): ''' Args: dash: as_of: Returns: ''' topp = [] omit = ['CFID', 'CFPARAMS', 'COMMENT', 'TEAM_NAME'] for team in dash: fixed_team = {k.lower(): v for k, v in team.items() if k not in omit} fixed_team.pop('team_name', None) fixed_team['as_of'] = convert_format(as_of, 'nba') topp.append(fixed_team) return topp