예제 #1
0
 def __init__(self, db, cookies=None, cache_name=None):
     '''
     Args:
         db:
         cookies:
         cache_name:
     '''
     logging.getLogger(__name__).addHandler(logging.NullHandler())
     self.scraper = FantasyLabsNBAScraper(cookies=cookies, cache_name=cache_name)
     self.parser = FantasyLabsNBAParser()
     if db:
         self.db = db
         self.insert_db = True
     else:
         self.insert_db=False
예제 #2
0
    def __init__(self, db=True, safe=True, use_cache=True):

        # see http://stackoverflow.com/questions/8134444
        NBAAgent.__init__(self)

        self.logger = logging.getLogger(__name__)

        if db:
            self.db = FantasyLabsNBAPg()

        self.safe = safe

        if use_cache:
            self.scraper = FantasyLabsNBAScraper(use_cache=use_cache)

        else:
            self.scraper = FantasyLabsNBAScraper()

        self.parser = FantasyLabsNBAParser()
예제 #3
0
class FantasyLabsNBAAgent(object):
    '''
    Performs script-like tasks using fantasylabs scraper, parser, and db module

    Examples:
        a = FantasyLabsNBAAgent(db=db, cache_name='flabs-agent')
        players = a.today_models()
    '''

    def __init__(self, db, cookies=None, cache_name=None):
        '''
        Args:
            db:
            cookies:
            cache_name:
        '''
        logging.getLogger(__name__).addHandler(logging.NullHandler())
        self.scraper = FantasyLabsNBAScraper(cookies=cookies, cache_name=cache_name)
        self.parser = FantasyLabsNBAParser()
        if db:
            self.db = db
            self.insert_db = True
        else:
            self.insert_db=False


    def one_model(self, model_day, model_name='default'):
        '''
        Gets list of player models for day

        Args:
            model_day (str): in %Y-%m-%d format
            model_name (str): default, cash, etc.
            insert_db (bool): true if want to insert models into database

        Returns:
            players (list): parsed model

        Examples:
            a = FantasyLabsNBAAgent()
            model = a.one_model(model_day='03_01_2016')
        '''
        model = self.scraper.model(model_day=model_day, model_name=model_name)
        players = self.parser.model(content=model, site='dk')

        # need to run through pipeline
        #if self.insert_db:
        #    self.db.insert_salaries(players)
        #return players


    def many_models(self, model_name='default', range_start=None, range_end=None, all_missing=False):
        '''
        TODO: is not implemented
        Gets list of player models for day

        Args:
            range_start (str): in %Y-%m-%d format
            range_end (str): in %Y-%m-%d format
            model_name (str): default, cash, etc.

        Returns:
            players (list): parsed model

        Examples:
            a = FantasyLabsNBAAgent()
            models = a.many_models(range_start='2016-03-01', range_end='2016-03-07')
            models = a.many_models(all_missing=True)
        '''
        models = []
        if all_missing:
            for d in self.db.select_list(missing_models()):
                daystr = datetostr(d, 'fl')
                models.append({
                    'game_date': daystr,
                    'data': self.one_model(model_day=daystr, model_name=model_name),
                    'model_name': model_name
                })
        else:
            for d in date_list(range_end, range_start):
                daystr = datetostr(d, 'fl')
                models.append({
                    'game_date': daystr,
                    'data': self.one_model(model_day=daystr, model_name=model_name),
                    'model_name': model_name
                })
        if self.insert_db:
            self.db.insert_models(models)
        return models


    def ownership(self, day=None, all_missing=False):
        '''
        Args:
            day(str): in mm_dd_YYYY format
            all_missing(bool): single day or all missing days in season?
        Returns:
            players(list): of player ownership dict
        '''
        if day:
            day = convert_format(day, 'fl')
            own = self.scraper.ownership(day)
            if self.insert_db:
                self.db.insert_ownership(own, convert_format(day, 'std'))
            return own
        elif all_missing:
            owns = {}
            for day in self.db.select_list(missing_ownership()):
                daystr = datetostr(day, 'fl')
                own = self.scraper.ownership(daystr)
                self.db.insert_ownership(own, convert_format(daystr, 'std'))
                owns[daystr] = own
            return owns
        else:
            raise ValueError('must provide day or set all_missing to True')


    def salaries(self, day=None, all_missing=False):
        '''
        Args:
            day(str): in mm_dd_YYYY format
        Returns:
            players(list): of player dict
        '''
        if day:
            sals = self.parser.dk_salaries(self.scraper.model(day), day)
            if self.insert_db and sals:
                self.db.insert_salaries(sals, game_date=convert_format(day, 'std'))
            return sals
        elif all_missing:
            salaries = {}
            for day in self.db.select_list(missing_salaries()):
                daystr = datetostr(day, 'fl')
                sals = self.parser.dk_salaries(self.scraper.model(daystr), daystr)
                salaries[datetostr(day, 'nba')] = sals
                logging.debug('got salaries for {}'.format(daystr))
                time.sleep(1)
            if self.insert_db and salaries:
                self.db.insert_salaries_dict(salaries)
            return salaries
        else:
            raise ValueError('must provide day or set all_missing to True')


    def today_model(self, model_name='default'):
        '''
        Gets list of player models for today's games

        Args:
            model_name (str): default, cash, etc.

        Returns:
            players (list): parsed model

        Examples:
            a = FantasyLabsNBAAgent()
            models = a.today_model()
        '''
        today = dt.datetime.strftime(dt.datetime.today(), '%m_%d_%Y')
        model = self.scraper.model(model_day=today, model_name=model_name)
        return self.parser.model(content=model, gamedate=today)


    def update_player_xref(self):
        '''
        Adds missing players to player_xref table and updates dfs_salaries afterwards
        '''
        self.db.update(update_dfs_salaries_ids())
        missing = self.db.select_dict(missing_salaries_ids(source='fantasylabs'))

        if missing:
            nbapq = """SELECT nbacom_player_id as id, display_first_last as n FROM players"""
            nbadict = {}
            nbacount = defaultdict(int)
            for p in self.db.select_dict(nbapq):
                nbadict[p['id']] = p['n']
                nbacount[p['n']] += 1

            # loop through missing players
            # filter out players with duplicate names - need to manually resolve those
            # then look for direct match where name is not duplicated
            # then try to match using names
            insq = """INSERT INTO player_xref (nbacom_player_id, source, source_player_id, source_player_name)
                      VALUES ({}, 'fantasylabs', {}, '{}');"""

            for p in missing:
                if nbacount[p['n']] > 1:
                    logging.error('need to manually resolve {}'.format(p))
                    continue
                match = [k for k,v in listitems(nbadict) if v == p['n']]
                if match:
                    self.db.update(insq.format(match[0], p['id'], p['n']))
                    logging.debug('added to xref: {}'.format(p))
                    continue
                match = [k for k,v in listitems(nbadict) if v == match_player(p['n'], list(nbadict.values()), threshold=.8)]
                if match:
                    self.db.update(insq.format(match[0], p['id'], p['n']))
                    logging.debug('added to xref: {}'.format(p))
                else:
                    logging.error('need to manually resolve {}'.format(p))

            # now update dfs_salaries nbacom_player_id from player_xref
            self.db.update(update_dfs_salaries_ids())

        else:
            logging.info('no missing ids in dfs_salaries')
예제 #4
0
class FantasyLabsNBAAgent(NBAAgent):
    '''
    Performs script-like tasks using fantasylabs scraper, parser, and db module
    Intended to replace standalone scripts so can use common API and tools

    Examples:
        a = FantasyLabsNBAAgent()
        players = a.today_models()
        players, pp_players = a.today_models()
    '''

    def __init__(self, db=True, safe=True, use_cache=True):

        # see http://stackoverflow.com/questions/8134444
        NBAAgent.__init__(self)

        self.logger = logging.getLogger(__name__)

        if db:
            self.db = FantasyLabsNBAPg()

        self.safe = safe

        if use_cache:
            self.scraper = FantasyLabsNBAScraper(use_cache=use_cache)

        else:
            self.scraper = FantasyLabsNBAScraper()

        self.parser = FantasyLabsNBAParser()

    def optimizer_pipeline(self, models):
        '''
        Takes fantasylabs models, make ready to create Player objects for pydfs_lineup_optimizer

        Args:
            models (list): is parsed json from day's models

        Returns:
            players (list): list of players, fixed for use in pydfs_lineup_optimizer

        Examples:
            a = FantasyLabsNBAAgent()
            models = a.today_models()
            players = a.optimizer_pipeline(models)

        '''

        fl_keys = ['PlayerId', 'Player_Name', 'Position', 'Team', 'Salary', 'Score', 'AvgPts', 'Ceiling', 'Floor', 'ProjPlusMinus']
        fl_players = [{k: v for k,v in p.iteritems() if k in fl_keys} for p in models]

        # remove null values
        for idx, flp in enumerate(fl_players):
            if flp.get('Ceiling') is None:
                fl_players[idx]['Ceiling'] = 0
            if flp.get('Floor') is None:
                fl_players[idx]['Floor'] = 0
            if flp.get('AvgPts') is None:
                fl_players[idx]['AvgPts'] = 0

        return fl_players

    def past_day_models(self, model_day, model_name='default', fn=None, insert_db=False):
        '''
        Gets list of player models for day

        Args:
            model_day (str): in %Y-%m-%d format
            model_name (str): default, cash, etc.
            fn (str): name of model json file to load from disk
            insert_db (bool): true if want to insert models into database

        Returns:
            players (list): parsed model
            pp_players (list): parsed model, prepared for insert into database

        Examples:
            a = FantasyLabsNBAAgent()
            models = a.past_day_models(model_day='2016-03-01')
            models = a.past_day_models(model_day='2016-03-01', model_name='phan')
            models = a.past_day_models(model_day='2016-03-01', model_name='phan', insert_db=True)

        '''

        if fn:
            with open(fn, 'r') as infile:
                model = json.load(infile)

        else:
            model = self.scraper.model(model_day=model_day, model_name=model_name)

        players = self.parser.model(content=model, site='dk', gamedate=model_day)
        pp_players = self.db.preprocess_salaries(players)

        if self.db and insert_db:
            self.db.insert_salaries(pp_players)

        return players, pp_players

    def range_models(self, range_start, range_end, model_name='default', insert_db=False):
        '''
        Gets list of player models for day

        Args:
            range_start (str): in %Y-%m-%d format
            range_end (str): in %Y-%m-%d format
            model_name (str): default, cash, etc.
            fn (str): name of model json file to load from disk
            insert_db (bool): true if want to insert models into database

        Returns:
            players (list): parsed model
            pp_players (list): parsed model, prepared for insert into database

        Examples:
            a = FantasyLabsNBAAgent()
            models = a.range_models(range_start='2016-03-01', range_end='2016-03-07')
            models = a.range_models(range_start='2016-03-01', range_end='2016-03-07', model_name='phan')
            models = a.range_models(range_start='2016-03-01', range_end='2016-03-07', model_name='phan', insert_db=True)
        '''

        players = []
        pp_players = []

        for d in date_list(range_end, range_start):
            d_players, d_pp_players = self.past_day_models(model_day=dt.datetime.strftime(d, '%Y-%m-%d'), model_name=model_name)
            players += d_players
            pp_players += d_pp_players

        if self.db and insert_db:
            self.db.insert_salaries(pp_players)

        return players, pp_players

    def today_games(self):
        '''
        Gets list of today's games

        Args:
            None

        Returns:
            list: parsed game json

        Examples:
            a = FantasyLabsNBAAgent()
            games = a.today_games()
        '''

        return self.parser.games(self.scraper.games_today())

    def today_models(self, model_name='default', fn=None, insert_db=False):
        '''
        Gets list of player models for today's games

        Args:
            model_name (str): default, cash, etc.
            fn (str): name of model json file to load from disk
            insert_db (bool): true if want to insert models into database

        Returns:
            players (list): parsed model
            pp_players (list): parsed model, prepared for insert into database

        Examples:
            a = FantasyLabsNBAAgent()
            models = a.today_models()
            models = a.today_models(model_name='phan')
            models = a.range_models(model_name='phan', insert_db=True)
        '''

        today = dt.datetime.strftime(dt.datetime.today(), '%Y-%m-%d')

        if fn:
            with open(fn, 'r') as infile:
                model = json.load(infile)

        else:
            model = self.scraper.model(model_day=today, model_name=model_name)

        players = self.parser.model(content=model, site='dk', gamedate=today)

        if self.db:
            pp_players = self.db.preprocess_salaries(players)

            if insert_db:
                self.db.insert_salaries(pp_players)

        else:
            pp_players = None

        return players, pp_players

    def update_models(self, season, model_name='default', insert_db=False):
        '''
        Fills in all missing models
        Query database for dates
        Then fetches missing dates

        Args:
            season (str): in YYYY-YY format
            model_name (str): default, cash, etc.
            insert_db (bool): true if want to insert models into database

        Returns:
            players (list): parsed model
            pp_players (list): parsed model, prepared for insert into database

        Examples:
            a = FantasyLabsNBAAgent()
            models = a.update_models(season='2015-16')
            models = a.update_models(season='2015-16', model_name='phan')
            models = a.update_models(season='2015-16', model_name='phan', insert_db=True)

        '''

        sql = "SELECT DISTINCT game_date FROM dfs.salaries WHERE source='fl' and dfs_site='dk'"
        # now execute it
        
        '''
        model = self.scraper.model(model_day=model_day, model_name=model_name)

        players = self.parser.model(content=model, site='dk', gamedate=model_day)
        pp_players = self.db.preprocess_salaries(players)

        if self.db and insert_db:
            self.db.insert_salaries(pp_players)
        '''
        
        return players, pp_players