def __init__(self, db, cookies=None, cache_name=None):
'''
Args:
db:
cookies:
cache_name:
'''
logging.getLogger(__name__).addHandler(logging.NullHandler())
self.scraper = FantasyLabsNBAScraper(cookies=cookies, cache_name=cache_name)
self.parser = FantasyLabsNBAParser()
if db:
self.db = db
self.insert_db = True
else:
self.insert_db=False
def __init__(self, db=True, safe=True, use_cache=True):
# see http://stackoverflow.com/questions/8134444
NBAAgent.__init__(self)
self.logger = logging.getLogger(__name__)
if db:
self.db = FantasyLabsNBAPg()
self.safe = safe
if use_cache:
self.scraper = FantasyLabsNBAScraper(use_cache=use_cache)
else:
self.scraper = FantasyLabsNBAScraper()
self.parser = FantasyLabsNBAParser()
class FantasyLabsNBAAgent(object):
'''
Performs script-like tasks using fantasylabs scraper, parser, and db module
Examples:
a = FantasyLabsNBAAgent(db=db, cache_name='flabs-agent')
players = a.today_models()
'''
def __init__(self, db, cookies=None, cache_name=None):
'''
Args:
db:
cookies:
cache_name:
'''
logging.getLogger(__name__).addHandler(logging.NullHandler())
self.scraper = FantasyLabsNBAScraper(cookies=cookies, cache_name=cache_name)
self.parser = FantasyLabsNBAParser()
if db:
self.db = db
self.insert_db = True
else:
self.insert_db=False
def one_model(self, model_day, model_name='default'):
'''
Gets list of player models for day
Args:
model_day (str): in %Y-%m-%d format
model_name (str): default, cash, etc.
insert_db (bool): true if want to insert models into database
Returns:
players (list): parsed model
Examples:
a = FantasyLabsNBAAgent()
model = a.one_model(model_day='03_01_2016')
'''
model = self.scraper.model(model_day=model_day, model_name=model_name)
players = self.parser.model(content=model, site='dk')
# need to run through pipeline
#if self.insert_db:
# self.db.insert_salaries(players)
#return players
def many_models(self, model_name='default', range_start=None, range_end=None, all_missing=False):
'''
TODO: is not implemented
Gets list of player models for day
Args:
range_start (str): in %Y-%m-%d format
range_end (str): in %Y-%m-%d format
model_name (str): default, cash, etc.
Returns:
players (list): parsed model
Examples:
a = FantasyLabsNBAAgent()
models = a.many_models(range_start='2016-03-01', range_end='2016-03-07')
models = a.many_models(all_missing=True)
'''
models = []
if all_missing:
for d in self.db.select_list(missing_models()):
daystr = datetostr(d, 'fl')
models.append({
'game_date': daystr,
'data': self.one_model(model_day=daystr, model_name=model_name),
'model_name': model_name
})
else:
for d in date_list(range_end, range_start):
daystr = datetostr(d, 'fl')
models.append({
'game_date': daystr,
'data': self.one_model(model_day=daystr, model_name=model_name),
'model_name': model_name
})
if self.insert_db:
self.db.insert_models(models)
return models
def ownership(self, day=None, all_missing=False):
'''
Args:
day(str): in mm_dd_YYYY format
all_missing(bool): single day or all missing days in season?
Returns:
players(list): of player ownership dict
'''
if day:
day = convert_format(day, 'fl')
own = self.scraper.ownership(day)
if self.insert_db:
self.db.insert_ownership(own, convert_format(day, 'std'))
return own
elif all_missing:
owns = {}
for day in self.db.select_list(missing_ownership()):
daystr = datetostr(day, 'fl')
own = self.scraper.ownership(daystr)
self.db.insert_ownership(own, convert_format(daystr, 'std'))
owns[daystr] = own
return owns
else:
raise ValueError('must provide day or set all_missing to True')
def salaries(self, day=None, all_missing=False):
'''
Args:
day(str): in mm_dd_YYYY format
Returns:
players(list): of player dict
'''
if day:
sals = self.parser.dk_salaries(self.scraper.model(day), day)
if self.insert_db and sals:
self.db.insert_salaries(sals, game_date=convert_format(day, 'std'))
return sals
elif all_missing:
salaries = {}
for day in self.db.select_list(missing_salaries()):
daystr = datetostr(day, 'fl')
sals = self.parser.dk_salaries(self.scraper.model(daystr), daystr)
salaries[datetostr(day, 'nba')] = sals
logging.debug('got salaries for {}'.format(daystr))
time.sleep(1)
if self.insert_db and salaries:
self.db.insert_salaries_dict(salaries)
return salaries
else:
raise ValueError('must provide day or set all_missing to True')
def today_model(self, model_name='default'):
'''
Gets list of player models for today's games
Args:
model_name (str): default, cash, etc.
Returns:
players (list): parsed model
Examples:
a = FantasyLabsNBAAgent()
models = a.today_model()
'''
today = dt.datetime.strftime(dt.datetime.today(), '%m_%d_%Y')
model = self.scraper.model(model_day=today, model_name=model_name)
return self.parser.model(content=model, gamedate=today)
def update_player_xref(self):
'''
Adds missing players to player_xref table and updates dfs_salaries afterwards
'''
self.db.update(update_dfs_salaries_ids())
missing = self.db.select_dict(missing_salaries_ids(source='fantasylabs'))
if missing:
nbapq = """SELECT nbacom_player_id as id, display_first_last as n FROM players"""
nbadict = {}
nbacount = defaultdict(int)
for p in self.db.select_dict(nbapq):
nbadict[p['id']] = p['n']
nbacount[p['n']] += 1
# loop through missing players
# filter out players with duplicate names - need to manually resolve those
# then look for direct match where name is not duplicated
# then try to match using names
insq = """INSERT INTO player_xref (nbacom_player_id, source, source_player_id, source_player_name)
VALUES ({}, 'fantasylabs', {}, '{}');"""
for p in missing:
if nbacount[p['n']] > 1:
logging.error('need to manually resolve {}'.format(p))
continue
match = [k for k,v in listitems(nbadict) if v == p['n']]
if match:
self.db.update(insq.format(match[0], p['id'], p['n']))
logging.debug('added to xref: {}'.format(p))
continue
match = [k for k,v in listitems(nbadict) if v == match_player(p['n'], list(nbadict.values()), threshold=.8)]
if match:
self.db.update(insq.format(match[0], p['id'], p['n']))
logging.debug('added to xref: {}'.format(p))
else:
logging.error('need to manually resolve {}'.format(p))
# now update dfs_salaries nbacom_player_id from player_xref
self.db.update(update_dfs_salaries_ids())
else:
logging.info('no missing ids in dfs_salaries')
class FantasyLabsNBAAgent(NBAAgent):
'''
Performs script-like tasks using fantasylabs scraper, parser, and db module
Intended to replace standalone scripts so can use common API and tools
Examples:
a = FantasyLabsNBAAgent()
players = a.today_models()
players, pp_players = a.today_models()
'''
def __init__(self, db=True, safe=True, use_cache=True):
# see http://stackoverflow.com/questions/8134444
NBAAgent.__init__(self)
self.logger = logging.getLogger(__name__)
if db:
self.db = FantasyLabsNBAPg()
self.safe = safe
if use_cache:
self.scraper = FantasyLabsNBAScraper(use_cache=use_cache)
else:
self.scraper = FantasyLabsNBAScraper()
self.parser = FantasyLabsNBAParser()
def optimizer_pipeline(self, models):
'''
Takes fantasylabs models, make ready to create Player objects for pydfs_lineup_optimizer
Args:
models (list): is parsed json from day's models
Returns:
players (list): list of players, fixed for use in pydfs_lineup_optimizer
Examples:
a = FantasyLabsNBAAgent()
models = a.today_models()
players = a.optimizer_pipeline(models)
'''
fl_keys = ['PlayerId', 'Player_Name', 'Position', 'Team', 'Salary', 'Score', 'AvgPts', 'Ceiling', 'Floor', 'ProjPlusMinus']
fl_players = [{k: v for k,v in p.iteritems() if k in fl_keys} for p in models]
# remove null values
for idx, flp in enumerate(fl_players):
if flp.get('Ceiling') is None:
fl_players[idx]['Ceiling'] = 0
if flp.get('Floor') is None:
fl_players[idx]['Floor'] = 0
if flp.get('AvgPts') is None:
fl_players[idx]['AvgPts'] = 0
return fl_players
def past_day_models(self, model_day, model_name='default', fn=None, insert_db=False):
'''
Gets list of player models for day
Args:
model_day (str): in %Y-%m-%d format
model_name (str): default, cash, etc.
fn (str): name of model json file to load from disk
insert_db (bool): true if want to insert models into database
Returns:
players (list): parsed model
pp_players (list): parsed model, prepared for insert into database
Examples:
a = FantasyLabsNBAAgent()
models = a.past_day_models(model_day='2016-03-01')
models = a.past_day_models(model_day='2016-03-01', model_name='phan')
models = a.past_day_models(model_day='2016-03-01', model_name='phan', insert_db=True)
'''
if fn:
with open(fn, 'r') as infile:
model = json.load(infile)
else:
model = self.scraper.model(model_day=model_day, model_name=model_name)
players = self.parser.model(content=model, site='dk', gamedate=model_day)
pp_players = self.db.preprocess_salaries(players)
if self.db and insert_db:
self.db.insert_salaries(pp_players)
return players, pp_players
def range_models(self, range_start, range_end, model_name='default', insert_db=False):
'''
Gets list of player models for day
Args:
range_start (str): in %Y-%m-%d format
range_end (str): in %Y-%m-%d format
model_name (str): default, cash, etc.
fn (str): name of model json file to load from disk
insert_db (bool): true if want to insert models into database
Returns:
players (list): parsed model
pp_players (list): parsed model, prepared for insert into database
Examples:
a = FantasyLabsNBAAgent()
models = a.range_models(range_start='2016-03-01', range_end='2016-03-07')
models = a.range_models(range_start='2016-03-01', range_end='2016-03-07', model_name='phan')
models = a.range_models(range_start='2016-03-01', range_end='2016-03-07', model_name='phan', insert_db=True)
'''
players = []
pp_players = []
for d in date_list(range_end, range_start):
d_players, d_pp_players = self.past_day_models(model_day=dt.datetime.strftime(d, '%Y-%m-%d'), model_name=model_name)
players += d_players
pp_players += d_pp_players
if self.db and insert_db:
self.db.insert_salaries(pp_players)
return players, pp_players
def today_games(self):
'''
Gets list of today's games
Args:
None
Returns:
list: parsed game json
Examples:
a = FantasyLabsNBAAgent()
games = a.today_games()
'''
return self.parser.games(self.scraper.games_today())
def today_models(self, model_name='default', fn=None, insert_db=False):
'''
Gets list of player models for today's games
Args:
model_name (str): default, cash, etc.
fn (str): name of model json file to load from disk
insert_db (bool): true if want to insert models into database
Returns:
players (list): parsed model
pp_players (list): parsed model, prepared for insert into database
Examples:
a = FantasyLabsNBAAgent()
models = a.today_models()
models = a.today_models(model_name='phan')
models = a.range_models(model_name='phan', insert_db=True)
'''
today = dt.datetime.strftime(dt.datetime.today(), '%Y-%m-%d')
if fn:
with open(fn, 'r') as infile:
model = json.load(infile)
else:
model = self.scraper.model(model_day=today, model_name=model_name)
players = self.parser.model(content=model, site='dk', gamedate=today)
if self.db:
pp_players = self.db.preprocess_salaries(players)
if insert_db:
self.db.insert_salaries(pp_players)
else:
pp_players = None
return players, pp_players
def update_models(self, season, model_name='default', insert_db=False):
'''
Fills in all missing models
Query database for dates
Then fetches missing dates
Args:
season (str): in YYYY-YY format
model_name (str): default, cash, etc.
insert_db (bool): true if want to insert models into database
Returns:
players (list): parsed model
pp_players (list): parsed model, prepared for insert into database
Examples:
a = FantasyLabsNBAAgent()
models = a.update_models(season='2015-16')
models = a.update_models(season='2015-16', model_name='phan')
models = a.update_models(season='2015-16', model_name='phan', insert_db=True)
'''
sql = "SELECT DISTINCT game_date FROM dfs.salaries WHERE source='fl' and dfs_site='dk'"
# now execute it
'''
model = self.scraper.model(model_day=model_day, model_name=model_name)
players = self.parser.model(content=model, site='dk', gamedate=model_day)
pp_players = self.db.preprocess_salaries(players)
if self.db and insert_db:
self.db.insert_salaries(pp_players)
'''
return players, pp_players