def setUp(self):
self.today = dt.strftime(dt.today(), '%-m_%-d_%Y')
self.config = ConfigParser()
configfn = os.path.join(os.path.expanduser('~'), '.pgcred')
self.config.read(configfn)
self.db = FantasyLabsNBAPg(username=self.config['nbadb']['username'],
password=self.config['nbadb']['password'],
database=self.config['nbadb']['database'])
self.a = FantasyLabsNBAAgent(cache_name='testflabs-nba', db=self.db)
class FantasyLabsNBAAgent_test(unittest.TestCase):
def setUp(self):
self.today = dt.strftime(dt.today(), '%-m_%-d_%Y')
self.config = ConfigParser()
configfn = os.path.join(os.path.expanduser('~'), '.pgcred')
self.config.read(configfn)
self.db = FantasyLabsNBAPg(username=self.config['nbadb']['username'],
password=self.config['nbadb']['password'],
database=self.config['nbadb']['database'])
self.a = FantasyLabsNBAAgent(cache_name='testflabs-nba', db=self.db)
def test_init(self):
# this should fail
# adb = FantasyLabsNBAAgent(db=True)
adb = FantasyLabsNBAAgent(cache_name='testnba', db=self.db)
self.assertIsNotNone(adb.scraper)
self.assertIsNotNone(adb.parser)
self.assertIsNotNone(adb.db)
def test_salaries(self):
model = self.a.scraper.model(self.today)
q = """SELECT DISTINCT source_player_id, nbacom_player_id FROM dfs_salaries WHERE source = 'fantasylabs'"""
allp = {
sal.get('source_player_id'): sal.get('nbacom_player_id')
for sal in self.db.select_dict(q)
}
for p in self.a.parser.model(model):
print(p.get('Player_Name'), allp.get(int(p.get('PlayerId', 0))))
'''
def __init__(self, db=True, safe=True, use_cache=True):
# see http://stackoverflow.com/questions/8134444
NBAAgent.__init__(self)
self.logger = logging.getLogger(__name__)
if db:
self.db = FantasyLabsNBAPg()
self.safe = safe
if use_cache:
self.scraper = FantasyLabsNBAScraper(use_cache=use_cache)
else:
self.scraper = FantasyLabsNBAScraper()
self.parser = FantasyLabsNBAParser()
def main():
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
config = ConfigParser()
configfn = os.path.join(os.path.expanduser('~'), '.pgcred')
config.read(configfn)
nbapg = NBAComPg(username=config['nbadb']['username'],
password=config['nbadb']['password'],
database=config['nbadb']['database'])
flpg = FantasyLabsNBAPg(username=config['nbadb']['username'],
password=config['nbadb']['password'],
database=config['nbadb']['database'])
cn = 'nba-agent-{}'.format(today())
a = NBAComAgent(cache_name=cn, cookies=None, db=nbapg)
a.scraper.delay = 1
season = '2016-17'
# get the rosters
q = """SELECT is_starter FROM starters WHERE nbacom_player_id = {}
ORDER BY game_date DESC LIMIT 1"""
incomplete = []
roster_url = 'http://data.nba.com/data/10s/prod/v1/2016/teams/{}/roster.json'
for urlcode in nbapg.select_list(today_team_url_codes()):
print(urlcode)
try:
roster = a.scraper.get_json(roster_url.format(urlcode))
for p in roster["league"]["standard"]["players"]:
pid = p.get('personId')
starter = a.db.select_scalar(q.format(pid))
print('{}, {}'.format(pid, starter))
except:
incomplete.append(roster_url.format(urlcode))
for i in incomplete:
try:
roster = a.scraper.get_json(roster_url.format(urlcode))
for p in roster["league"]["standard"]["players"]:
starter = a.db.select_scalar(q.format(p))
print('{}, {}'.format(p, starter))
except:
logging.error('could not get {}'.format(i))
from nba.agents.fantasylabs import FantasyLabsNBAAgent
from nba.dates import datetostr
from nba.db.fantasylabs import FantasyLabsNBAPg
logger = logging.getLogger('nbadb-update')
hdlr = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.INFO)
logger.propagate = False
config = ConfigParser()
configfn = os.path.join(os.path.expanduser('~'), '.nbadb')
config.read(configfn)
flpg = FantasyLabsNBAPg(username=config['nbadb']['username'],
password=config['nbadb']['password'],
database=config['nbadb']['database'])
fla = FantasyLabsNBAAgent(db=flpg, cache_name='flabs-nba', cookies=browsercookie.firefox())
fla.update_player_xref()
q = """select distinct game_date from games where season = 2015 AND season_type = 'regular' order by game_date DESC"""
for d in flpg.select_list(q):
try:
fla.salaries(day=datetostr(d, site='fl'))
logger.info('completed {}'.format(d))
except Exception as e:
logger.exception('{} failed: {}'.format(d, e))
finally:
time.sleep(1.5)
def main():
#logger = logging.getLogger('nbadb-update')
#hdlr = logging.StreamHandler(sys.stdout)
#formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
#hdlr.setFormatter(formatter)
#logger.addHandler(hdlr)
#logger.setLevel(logging.INFO)
#logger.propagate = False
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
config = ConfigParser()
configfn = os.path.join(os.path.expanduser('~'), '.nbadb')
config.read(configfn)
nbapg = NBAComPg(username=config['nbadb']['username'],
password=config['nbadb']['password'],
database=config['nbadb']['database'])
flpg = FantasyLabsNBAPg(username=config['nbadb']['username'],
password=config['nbadb']['password'],
database=config['nbadb']['database'])
cn = 'nba-agent-{}'.format(today())
a = NBAComAgent(cache_name=cn, cookies=None, db=nbapg)
a.scraper.delay = 2
season = '2016-17'
# ensures players table is up-to-date before inserting gamelogs, etc.
a.new_players(season)
logging.info('finished update nba.com players')
# gets all missing (days) salaries from current seasons
from nba.agents.fantasylabs import FantasyLabsNBAAgent
fla = FantasyLabsNBAAgent(db=flpg, cache_name='flabs-nba')
fla.salaries(all_missing=True)
logging.info('finished dfs salaries')
# ensures that player_xref table includes all players from salaries
fla.update_player_xref()
logging.info('finished update player_xref')
# gets ownership data from fantasylabs
fla.ownership(all_missing=True)
logging.info('finished dfs ownership')
# player_gamelogs
a.player_gamelogs(season)
logging.info('finished nba.com player gamelogs')
# playerstats_daily
ps = a.playerstats(season, all_missing=True)
logging.info('finished playerstats daily')
# update team_gamelogs
a.team_gamelogs(season)
logging.info('finished team gamelogs')
# teamstats_daily
a.teamstats(season, all_missing=True)
logging.info('finished teamstats daily')
# team_opponent_dashboards
a.team_opponent_dashboards(season, all_missing=True)
logging.info('finished team_opponent_dashboards')
# refresh all materialized views
refreshq = """SELECT RefreshAllMaterializedViews('*');"""
nbapg.execute(refreshq)
def test_init(self):
flpg = FantasyLabsNBAPg()
def setUp(self):
self.db = FantasyLabsNBAPg()
class FantasyLabsNBAAgent(NBAAgent):
'''
Performs script-like tasks using fantasylabs scraper, parser, and db module
Intended to replace standalone scripts so can use common API and tools
Examples:
a = FantasyLabsNBAAgent()
players = a.today_models()
players, pp_players = a.today_models()
'''
def __init__(self, db=True, safe=True, use_cache=True):
# see http://stackoverflow.com/questions/8134444
NBAAgent.__init__(self)
self.logger = logging.getLogger(__name__)
if db:
self.db = FantasyLabsNBAPg()
self.safe = safe
if use_cache:
self.scraper = FantasyLabsNBAScraper(use_cache=use_cache)
else:
self.scraper = FantasyLabsNBAScraper()
self.parser = FantasyLabsNBAParser()
def optimizer_pipeline(self, models):
'''
Takes fantasylabs models, make ready to create Player objects for pydfs_lineup_optimizer
Args:
models (list): is parsed json from day's models
Returns:
players (list): list of players, fixed for use in pydfs_lineup_optimizer
Examples:
a = FantasyLabsNBAAgent()
models = a.today_models()
players = a.optimizer_pipeline(models)
'''
fl_keys = ['PlayerId', 'Player_Name', 'Position', 'Team', 'Salary', 'Score', 'AvgPts', 'Ceiling', 'Floor', 'ProjPlusMinus']
fl_players = [{k: v for k,v in p.iteritems() if k in fl_keys} for p in models]
# remove null values
for idx, flp in enumerate(fl_players):
if flp.get('Ceiling') is None:
fl_players[idx]['Ceiling'] = 0
if flp.get('Floor') is None:
fl_players[idx]['Floor'] = 0
if flp.get('AvgPts') is None:
fl_players[idx]['AvgPts'] = 0
return fl_players
def past_day_models(self, model_day, model_name='default', fn=None, insert_db=False):
'''
Gets list of player models for day
Args:
model_day (str): in %Y-%m-%d format
model_name (str): default, cash, etc.
fn (str): name of model json file to load from disk
insert_db (bool): true if want to insert models into database
Returns:
players (list): parsed model
pp_players (list): parsed model, prepared for insert into database
Examples:
a = FantasyLabsNBAAgent()
models = a.past_day_models(model_day='2016-03-01')
models = a.past_day_models(model_day='2016-03-01', model_name='phan')
models = a.past_day_models(model_day='2016-03-01', model_name='phan', insert_db=True)
'''
if fn:
with open(fn, 'r') as infile:
model = json.load(infile)
else:
model = self.scraper.model(model_day=model_day, model_name=model_name)
players = self.parser.model(content=model, site='dk', gamedate=model_day)
pp_players = self.db.preprocess_salaries(players)
if self.db and insert_db:
self.db.insert_salaries(pp_players)
return players, pp_players
def range_models(self, range_start, range_end, model_name='default', insert_db=False):
'''
Gets list of player models for day
Args:
range_start (str): in %Y-%m-%d format
range_end (str): in %Y-%m-%d format
model_name (str): default, cash, etc.
fn (str): name of model json file to load from disk
insert_db (bool): true if want to insert models into database
Returns:
players (list): parsed model
pp_players (list): parsed model, prepared for insert into database
Examples:
a = FantasyLabsNBAAgent()
models = a.range_models(range_start='2016-03-01', range_end='2016-03-07')
models = a.range_models(range_start='2016-03-01', range_end='2016-03-07', model_name='phan')
models = a.range_models(range_start='2016-03-01', range_end='2016-03-07', model_name='phan', insert_db=True)
'''
players = []
pp_players = []
for d in date_list(range_end, range_start):
d_players, d_pp_players = self.past_day_models(model_day=dt.datetime.strftime(d, '%Y-%m-%d'), model_name=model_name)
players += d_players
pp_players += d_pp_players
if self.db and insert_db:
self.db.insert_salaries(pp_players)
return players, pp_players
def today_games(self):
'''
Gets list of today's games
Args:
None
Returns:
list: parsed game json
Examples:
a = FantasyLabsNBAAgent()
games = a.today_games()
'''
return self.parser.games(self.scraper.games_today())
def today_models(self, model_name='default', fn=None, insert_db=False):
'''
Gets list of player models for today's games
Args:
model_name (str): default, cash, etc.
fn (str): name of model json file to load from disk
insert_db (bool): true if want to insert models into database
Returns:
players (list): parsed model
pp_players (list): parsed model, prepared for insert into database
Examples:
a = FantasyLabsNBAAgent()
models = a.today_models()
models = a.today_models(model_name='phan')
models = a.range_models(model_name='phan', insert_db=True)
'''
today = dt.datetime.strftime(dt.datetime.today(), '%Y-%m-%d')
if fn:
with open(fn, 'r') as infile:
model = json.load(infile)
else:
model = self.scraper.model(model_day=today, model_name=model_name)
players = self.parser.model(content=model, site='dk', gamedate=today)
if self.db:
pp_players = self.db.preprocess_salaries(players)
if insert_db:
self.db.insert_salaries(pp_players)
else:
pp_players = None
return players, pp_players
def update_models(self, season, model_name='default', insert_db=False):
'''
Fills in all missing models
Query database for dates
Then fetches missing dates
Args:
season (str): in YYYY-YY format
model_name (str): default, cash, etc.
insert_db (bool): true if want to insert models into database
Returns:
players (list): parsed model
pp_players (list): parsed model, prepared for insert into database
Examples:
a = FantasyLabsNBAAgent()
models = a.update_models(season='2015-16')
models = a.update_models(season='2015-16', model_name='phan')
models = a.update_models(season='2015-16', model_name='phan', insert_db=True)
'''
sql = "SELECT DISTINCT game_date FROM dfs.salaries WHERE source='fl' and dfs_site='dk'"
# now execute it
'''
model = self.scraper.model(model_day=model_day, model_name=model_name)
players = self.parser.model(content=model, site='dk', gamedate=model_day)
pp_players = self.db.preprocess_salaries(players)
if self.db and insert_db:
self.db.insert_salaries(pp_players)
'''
return players, pp_players
def main():
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
config = ConfigParser()
configfn = os.path.join(os.path.expanduser('~'), '.pgcred')
config.read(configfn)
nbapg = NBAComPg(username=config['nbadb']['username'],
password=config['nbadb']['password'],
database=config['nbadb']['database'])
flpg = FantasyLabsNBAPg(username=config['nbadb']['username'],
password=config['nbadb']['password'],
database=config['nbadb']['database'])
fla = FantasyLabsNBAAgent(db=flpg, cache_name='flabs-nba', cookies=browsercookie.firefox())
rgurua = RotoGuruNBAAgent(db=nbapg, cache_name='rg-nba')
cn = 'nba-agent-{}'.format(today())
a = NBAComAgent(cache_name=cn, cookies=None, db=nbapg)
a.scraper.delay = 1
season = '2016-17'
# ensures players table is up-to-date before inserting gamelogs, etc.
logging.info('starting update nba.com players')
players = a.new_players(season[0:4])
logging.info('finished update nba.com players')
# gets all missing (days) salaries from current seasons
logging.info('starting dfs salaries')
fla.salaries(all_missing=True)
rgurua.salaries(all_missing=True)
logging.info('finished dfs salaries')
# ensures that player_xref table includes all players from salaries
#logging.info('starting update player_xref')
#fla.update_player_xref()
#logging.info('finished update player_xref')
# gets model from fantasylabs
#td = today('fl')
#mn = 'phan'
#flpg.insert_models([{
# 'game_date': today('fl'),
# 'data': fla.scraper.model(model_day=td, model_name=mn),
# 'model_name': mn}])
# gets ownership data from fantasylabs
#logging.info('starting dfs ownership')
#fla.ownership(all_missing=True)
#logging.info('finished dfs ownership')
# gets data from rotogrinders
#logging.info('starting rotogrinders')
#rs = RotoGrindersNBAScraper()
#rp = RotoGrindersNBAParser()
#rdb = RotoGrindersNBAPg(username=config['nbadb']['username'],
# password=config['nbadb']['password'],
# database=config['nbadb']['database'])
#jsonstr = rp.odds(rs.odds())
#rdb.insert_odds(today(), json.loads(jsonstr))
#logging.info('finished rotogrinders')
# player_gamelogs
logging.info('starting nba.com player gamelogs')
a.player_gamelogs(season)
logging.info('finished nba.com player gamelogs')
# playerstats_daily
logging.info('starting playerstats daily')
ps = a.playerstats(season, all_missing=True)
logging.info('finished playerstats daily')
# player_boxscores_combined
logging.info('starting player_boxscores_combined')
pbs = a.player_boxscores_combined()
logging.info('finished player_boxscores_combined')
# update team_gamelogs
logging.info('starting team gamelogs')
a.team_gamelogs(season)
logging.info('finished team gamelogs')
# teamstats_daily
logging.info('starting teamstats daily')
a.teamstats(season, all_missing=True)
logging.info('finished teamstats daily')
# team_boxscores_combined
logging.info('start team_boxscores_combined')
tbs = a.team_boxscores_combined()
logging.info('finished team_boxscores_combined')
# team_opponent_dashboards
logging.info('start team_opponent_dashboards')
a.team_opponent_dashboards(season, all_missing=True)
logging.info('finished team_opponent_dashboards')
# v2015 boxscores - linescores, refs, etc.
logging.info('start linescores')
a.linescores()
logging.info('finished linescores')
# odds and lines
logging.info('start odds and lines')
dba = DonBestNBAAgent(db=nbapg)
odds = dba.odds(all_missing=True)
logging.debug(odds)
logging.info('finished odds and lines')
# refresh all materialized views
logging.info('start refresh materialized queries')
nbapg.refresh_materialized()
logging.info('refreshed materialized queries')
