def missing_gamelog_players(season_code):
'''
Gets list of players from gamelogs that are not in player table
Args:
a (NBAComAgent): object instance
Returns:
list: of players from gamelogs that are not in player table
'''
start = datetostr(season_start(season_code=season_code), fmt='nba')
content = a.scraper.season_gamelogs(
season_code, 'P',
date_from=start,
date_to=today(fmt='nba'))
gamelogs = a.parser.season_gamelogs(content, 'P')
wanted = ["PLAYER_ID", "PLAYER_NAME", "TEAM_ID", "TEAM_ABBREVIATION",
"GAME_ID", "GAME_DATE", "MATCHUP"]
players = [{k: v for k, v in p.items() if k in wanted} for p in gamelogs]
currids = set([p['PLAYER_ID'] for p in players])
allids = set(a.db.select_list('SELECT nbacom_player_id from player'))
missing = currids - allids
msg = 'there are {} mising players from gamelogs'.format(len(missing))
logging.info(msg)
return [p for p in players if p.get('PLAYER_ID') in missing]
def get_wayback(self, url, d=None, max_delta=None):
'''
Gets page from the wayback machine
Args:
url: of the site you want, not the wayback machine
d: datestring, if None then get most recent one
max_delta: int, how many days off can the last page be from the requested date
Returns:
content: HTML string
'''
content = None
if not d:
d = today('db')
else:
d = convert_format(d, 'db')
resp = self.get_json(self.wburl.format(url, d))
if resp:
ts = resp['archived_snapshots']['closest']['timestamp'][:8]
if max_delta:
closest_url = resp['archived_snapshots']['closest']['url']
if subtract_datestr(d, ts) <= max_delta:
content = self.get(
resp['archived_snapshots']['closest']['url'])
else:
logging.error('page is too old: {}'.format(ts))
else:
closest_url = resp['archived_snapshots']['closest']['url']
return self.get(closest_url)
else:
logging.error('url unavailable on wayback machine')
return content, ts
def team_gamelogs_table(tgl):
'''
Cleans merged list of team gamelogs base + advanced
Arguments:
tgl: list of dictionaries
Returns:
cleaned_items(list)
'''
omit = ['matchup', 'season_id', 'team_name', 'video_available', 'wl']
cleaned_items = []
# skip today - no reliable way of testing if game is over
for gl in [l for l in tgl if l['GAME_DATE'] != today()]:
cleaned_item = {k.lower(): v for k,v in gl.items() if k.lower() not in omit}
if 'game_id' in cleaned_item:
cleaned_item['nbacom_game_id'] = cleaned_item['game_id']
cleaned_item.pop('game_id', None)
if 'team_id' in cleaned_item:
cleaned_item['nbacom_team_id'] = cleaned_item['team_id']
cleaned_item.pop('team_id', None)
if cleaned_item.get('team_abbreviation'):
cleaned_item['team_code'] = cleaned_item['team_abbreviation']
cleaned_item.pop('team_abbreviation', None)
if cleaned_item.get('min'):
cleaned_item['minutes'] = cleaned_item['min']
cleaned_item.pop('min', None)
cleaned_items.append(cleaned_item)
return cleaned_items
def main():
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
config = ConfigParser()
configfn = os.path.join(os.path.expanduser('~'), '.pgcred')
config.read(configfn)
cn = 'fl-agent-{}'.format(today())
fla = FantasyLabsNBAAgent(db=None,
cache_name=cn,
cookies=browsercookie.firefox())
models = fla.one_model(today(fmt='fl'), 'phan')
optimizer = LineupOptimizer(settings.DraftKingsBasketballSettings)
optimizer._players = fl_to_pydfs(models, weights=[.1, .6, .3])
for lineup in optimizer.optimize(n=5):
print lineup
def main():
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
config = ConfigParser()
configfn = os.path.join(os.path.expanduser('~'), '.pgcred')
config.read(configfn)
nbapg = NBAComPg(username=config['nbadb']['username'],
password=config['nbadb']['password'],
database=config['nbadb']['database'])
flpg = FantasyLabsNBAPg(username=config['nbadb']['username'],
password=config['nbadb']['password'],
database=config['nbadb']['database'])
cn = 'nba-agent-{}'.format(today())
a = NBAComAgent(cache_name=cn, cookies=None, db=nbapg)
a.scraper.delay = 1
season = '2016-17'
# get the rosters
q = """SELECT is_starter FROM starters WHERE nbacom_player_id = {}
ORDER BY game_date DESC LIMIT 1"""
incomplete = []
roster_url = 'http://data.nba.com/data/10s/prod/v1/2016/teams/{}/roster.json'
for urlcode in nbapg.select_list(today_team_url_codes()):
print(urlcode)
try:
roster = a.scraper.get_json(roster_url.format(urlcode))
for p in roster["league"]["standard"]["players"]:
pid = p.get('personId')
starter = a.db.select_scalar(q.format(pid))
print('{}, {}'.format(pid, starter))
except:
incomplete.append(roster_url.format(urlcode))
for i in incomplete:
try:
roster = a.scraper.get_json(roster_url.format(urlcode))
for p in roster["league"]["standard"]["players"]:
starter = a.db.select_scalar(q.format(p))
print('{}, {}'.format(p, starter))
except:
logging.error('could not get {}'.format(i))
def player_gamelogs_table(gl):
'''
Prepares players for insertion into nbadb players table
Args:
gl: list of dict
Returns:
List of dict formatted for insertion into database
'''
fixed = []
omit = ['VIDEO_AVAILABLE', 'TEAM_NAME', 'MATCHUP', 'WL', 'SEASON_ID']
# skip today - no reliable way of testing if game is over
for l in gl:
if l.get('GAME_DATE') == today():
continue
cl = {k.lower().strip(): v for k, v in l.items() if k.upper() not in omit}
cl['dk_points'] = dk_points(cl)
cl['fd_points'] = fd_points(cl)
# change to nbacom_player_id
if 'game_id' in cl:
cl['nbacom_game_id'] = cl['game_id']
cl.pop('game_id', None)
if 'player_id' in cl:
cl['nbacom_player_id'] = cl['player_id']
cl.pop('player_id', None)
if 'team_id' in cl:
cl['nbacom_team_id'] = cl['team_id']
cl.pop('team_id', None)
if 'team_abbreviation' in cl:
cl['team_code'] = cl['team_abbreviation']
cl.pop('team_abbreviation', None)
fixed.append(cl)
return fixed
def main():
#logger = logging.getLogger('nbadb-update')
#hdlr = logging.StreamHandler(sys.stdout)
#formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
#hdlr.setFormatter(formatter)
#logger.addHandler(hdlr)
#logger.setLevel(logging.INFO)
#logger.propagate = False
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
config = ConfigParser()
configfn = os.path.join(os.path.expanduser('~'), '.nbadb')
config.read(configfn)
nbapg = NBAComPg(username=config['nbadb']['username'],
password=config['nbadb']['password'],
database=config['nbadb']['database'])
flpg = FantasyLabsNBAPg(username=config['nbadb']['username'],
password=config['nbadb']['password'],
database=config['nbadb']['database'])
cn = 'nba-agent-{}'.format(today())
a = NBAComAgent(cache_name=cn, cookies=None, db=nbapg)
a.scraper.delay = 2
season = '2016-17'
# ensures players table is up-to-date before inserting gamelogs, etc.
a.new_players(season)
logging.info('finished update nba.com players')
# gets all missing (days) salaries from current seasons
from nba.agents.fantasylabs import FantasyLabsNBAAgent
fla = FantasyLabsNBAAgent(db=flpg, cache_name='flabs-nba')
fla.salaries(all_missing=True)
logging.info('finished dfs salaries')
# ensures that player_xref table includes all players from salaries
fla.update_player_xref()
logging.info('finished update player_xref')
# gets ownership data from fantasylabs
fla.ownership(all_missing=True)
logging.info('finished dfs ownership')
# player_gamelogs
a.player_gamelogs(season)
logging.info('finished nba.com player gamelogs')
# playerstats_daily
ps = a.playerstats(season, all_missing=True)
logging.info('finished playerstats daily')
# update team_gamelogs
a.team_gamelogs(season)
logging.info('finished team gamelogs')
# teamstats_daily
a.teamstats(season, all_missing=True)
logging.info('finished teamstats daily')
# team_opponent_dashboards
a.team_opponent_dashboards(season, all_missing=True)
logging.info('finished team_opponent_dashboards')
# refresh all materialized views
refreshq = """SELECT RefreshAllMaterializedViews('*');"""
nbapg.execute(refreshq)
def run():
'''
Updates nba.com statistics
Args:
None
Returns:
None
'''
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
db = getdb()
cn = 'nba-agent-{}'.format(today())
a = NBAComAgent(cache_name=cn, db=db)
a.scraper.delay = 1
season_year = 2018
season_code = '2017-18'
# ensures players table is up-to-date before inserting gamelogs, etc.
# players uses 2017 as season_year if season_code is 2017-18
# whereas nbadb calls that season_year 2018
logging.info('starting update nba.com players')
nbadb_player_update.run()
logging.info('finished update nba.com players')
# player_gamelogs
logging.info('starting nba.com player gamelogs')
a.player_gamelogs(season_code,
date_from=datetostr(
season_start(season_code=season_code), fmt='nba'),
date_to=yesterday(fmt='nba'))
logging.info('finished nba.com player gamelogs')
# playerstats_daily
logging.info('starting playerstats daily')
a.playerstats(season_code, all_missing=True, per_mode='Totals')
a.playerstats(season_code, all_missing=True, per_mode='PerGame')
logging.info('finished playerstats daily')
# player and team boxscores combined
logging.info('starting player_boxscores_combined')
pbs, tbs = a.combined_boxscores()
logging.info('finished player_boxscores_combined')
# update team_gamelogs
logging.info('starting team gamelogs')
a.team_gamelogs(season_code)
logging.info('finished team gamelogs')
# teamstats_daily
logging.info('starting teamstats daily')
a.teamstats(season_code, all_missing=True, per_mode='Totals')
a.teamstats(season_code, all_missing=True, per_mode='PerGame')
logging.info('finished teamstats daily')
# team_opponent_dashboards
logging.info('start team_opponent_dashboards')
a.team_opponent_dashboards(season_code,
all_missing=True,
per_mode='Totals')
a.team_opponent_dashboards(season_code,
all_missing=True,
per_mode='PerGame')
logging.info('finished team_opponent_dashboards')
# game boxscores - linescores, refs, etc.
logging.info('start game_boxscores')
a.game_boxscores()
logging.info('finished game_boxscores')
# refresh all materialized views
logging.info('start refresh materialized views')
a.refresh_materialized()
logging.info('refreshed materialized views')
def run():
'''
Updates nba.com statistics
Args:
None
Returns:
None
'''
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
db = getdb()
cn = 'nba-agent-{}'.format(today())
a = NBAComAgent(cache_name=cn, db=db)
a.scraper.delay = 1
season_year = 2018
season_code = '2017-18'
# gets all missing (days) salaries from current seasons
#logging.info('starting dfs salaries')
#fla.salaries(all_missing=True)
#rgurua.salaries(all_missing=True)
#logging.info('finished dfs salaries')
# ensures that player_xref table includes all players from salaries
#logging.info('starting update player_xref')
#fla.update_player_xref()
#logging.info('finished update player_xref')
# gets model from fantasylabs
#td = today('fl')
#mn = 'phan'
#flpg.insert_models([{
# 'game_date': today('fl'),
# 'data': fla.scraper.model(model_day=td, model_name=mn),
# 'model_name': mn}])
# gets ownership data from fantasylabs
#logging.info('starting dfs ownership')
#fla.ownership(all_missing=True)
#logging.info('finished dfs ownership')
# gets data from rotogrinders
#logging.info('starting rotogrinders')
#rs = RotoGrindersNBAScraper()
#rp = RotoGrindersNBAParser()
#rdb = RotoGrindersNBAPg(username=config['nbadb']['username'],
# password=config['nbadb']['password'],
# database=config['nbadb']['database'])
#jsonstr = rp.odds(rs.odds())
#rdb.insert_odds(today(), json.loads(jsonstr))
#logging.info('finished rotogrinders')
# refresh all materialized views
logging.info('start refresh materialized queries')
a.refresh_materialized()
logging.info('refreshed materialized queries')
# odds and lines
logging.info('start odds and lines')
dba = DonBestNBAAgent(db=db)
dba.odds(all_missing=True)
logging.info('finished odds and lines')
def main():
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
config = ConfigParser()
configfn = os.path.join(os.path.expanduser('~'), '.pgcred')
config.read(configfn)
nbapg = NBAComPg(username=config['nbadb']['username'],
password=config['nbadb']['password'],
database=config['nbadb']['database'])
flpg = FantasyLabsNBAPg(username=config['nbadb']['username'],
password=config['nbadb']['password'],
database=config['nbadb']['database'])
fla = FantasyLabsNBAAgent(db=flpg, cache_name='flabs-nba', cookies=browsercookie.firefox())
rgurua = RotoGuruNBAAgent(db=nbapg, cache_name='rg-nba')
cn = 'nba-agent-{}'.format(today())
a = NBAComAgent(cache_name=cn, cookies=None, db=nbapg)
a.scraper.delay = 1
season = '2016-17'
# ensures players table is up-to-date before inserting gamelogs, etc.
logging.info('starting update nba.com players')
players = a.new_players(season[0:4])
logging.info('finished update nba.com players')
# gets all missing (days) salaries from current seasons
logging.info('starting dfs salaries')
fla.salaries(all_missing=True)
rgurua.salaries(all_missing=True)
logging.info('finished dfs salaries')
# ensures that player_xref table includes all players from salaries
#logging.info('starting update player_xref')
#fla.update_player_xref()
#logging.info('finished update player_xref')
# gets model from fantasylabs
#td = today('fl')
#mn = 'phan'
#flpg.insert_models([{
# 'game_date': today('fl'),
# 'data': fla.scraper.model(model_day=td, model_name=mn),
# 'model_name': mn}])
# gets ownership data from fantasylabs
#logging.info('starting dfs ownership')
#fla.ownership(all_missing=True)
#logging.info('finished dfs ownership')
# gets data from rotogrinders
#logging.info('starting rotogrinders')
#rs = RotoGrindersNBAScraper()
#rp = RotoGrindersNBAParser()
#rdb = RotoGrindersNBAPg(username=config['nbadb']['username'],
# password=config['nbadb']['password'],
# database=config['nbadb']['database'])
#jsonstr = rp.odds(rs.odds())
#rdb.insert_odds(today(), json.loads(jsonstr))
#logging.info('finished rotogrinders')
# player_gamelogs
logging.info('starting nba.com player gamelogs')
a.player_gamelogs(season)
logging.info('finished nba.com player gamelogs')
# playerstats_daily
logging.info('starting playerstats daily')
ps = a.playerstats(season, all_missing=True)
logging.info('finished playerstats daily')
# player_boxscores_combined
logging.info('starting player_boxscores_combined')
pbs = a.player_boxscores_combined()
logging.info('finished player_boxscores_combined')
# update team_gamelogs
logging.info('starting team gamelogs')
a.team_gamelogs(season)
logging.info('finished team gamelogs')
# teamstats_daily
logging.info('starting teamstats daily')
a.teamstats(season, all_missing=True)
logging.info('finished teamstats daily')
# team_boxscores_combined
logging.info('start team_boxscores_combined')
tbs = a.team_boxscores_combined()
logging.info('finished team_boxscores_combined')
# team_opponent_dashboards
logging.info('start team_opponent_dashboards')
a.team_opponent_dashboards(season, all_missing=True)
logging.info('finished team_opponent_dashboards')
# v2015 boxscores - linescores, refs, etc.
logging.info('start linescores')
a.linescores()
logging.info('finished linescores')
# odds and lines
logging.info('start odds and lines')
dba = DonBestNBAAgent(db=nbapg)
odds = dba.odds(all_missing=True)
logging.debug(odds)
logging.info('finished odds and lines')
# refresh all materialized views
logging.info('start refresh materialized queries')
nbapg.refresh_materialized()
logging.info('refreshed materialized queries')