Beispiel #1
0
class NBAComAgent(object):
    '''
    Performs script-like tasks using NBA.com API
    '''
    def __init__(self, db=None, cache_name=None, cookies=None):
        '''
        Arguments:
            cache_name: str for scraper cache_name
            cookies: cookie jar
            db: NBAComPg instance
        '''
        logging.getLogger(__name__).addHandler(logging.NullHandler())
        self.scraper = NBAComScraper(cache_name=cache_name, cookies=cookies)
        self.parser = NBAComParser()
        if db:
            self.db = db
            self.insert_db = True
        else:
            self.insert_db = False

    def _combined_player_boxscores(self, gid):
        '''
        Combines 5 types of boxscores from nba.com API into list of boxscores
        Arguments:
            gid: string game ID, with leading '00'
        Returns:
            list of player boxscores combined - traditional, advanced, misc, scoring, usage
        Examples:
            a = NBAComAgent()
            playerboxes = a.combined_player_boxscores('0020161001')
        '''

        traditional_players, traditional_teams, traditional_starter_bench = self.parser.boxscore_traditional(
            self.scraper.boxscore_traditional(gid))
        adv_players, adv_team = self.parser.boxscore_advanced(
            self.scraper.boxscore_advanced(gid))
        misc_players, misc_team = self.parser.boxscore_misc(
            self.scraper.boxscore_misc(gid))
        scoring_players, scoring_team = self.parser.boxscore_scoring(
            self.scraper.boxscore_scoring(gid))
        usage_players = self.parser.boxscore_usage(
            self.scraper.boxscore_usage(gid))

        # now need to combine player and team boxscores
        players = merge(
            dict(),
            [{t['PLAYER_ID']: t
              for t in traditional_players},
             {t['PLAYER_ID']: t
              for t in adv_players}, {t['PLAYER_ID']: t
                                      for t in misc_players},
             {t['PLAYER_ID']: t
              for t in scoring_players},
             {t['PLAYER_ID']: t
              for t in usage_players}])
        return players.values()

    def combined_team_boxscores(self, gid):
        '''
        Combines 5 types of boxscores from nba.com API into list of boxscores
        Arguments:
            gid: string game ID, with leading '00'
        Returns:
            list of team boxscores - combined traditional, advanced, misc, scoring
        Examples:
            a = NBAComAgent()
            teamboxes = a.combined_team_boxscores('0020161001')
        '''
        traditional_players, traditional_teams, traditional_starter_bench = self.parser.boxscore_traditional(
            self.scraper.boxscore_traditional(gid))
        adv_players, adv_teams = self.parser.boxscore_advanced(
            self.scraper.boxscore_advanced(gid))
        misc_players, misc_teams = self.parser.boxscore_misc(
            self.scraper.boxscore_misc(gid))
        scoring_players, scoring_teams = self.parser.boxscore_scoring(
            self.scraper.boxscore_scoring(gid))

        # now need to combine player and team boxscores
        teams = merge(
            dict(),
            [{t['TEAM_ID']: t
              for t in traditional_teams},
             {t['TEAM_ID']: t
              for t in adv_teams}, {t['TEAM_ID']: t
                                    for t in misc_teams},
             {t['TEAM_ID']: t
              for t in scoring_teams}])
        return list(teams.values())

    def linescores(self):
        '''
        Updates gamesmeta table with game_information
        '''
        #q = """SELECT '00' || game_id, to_char(game_date, 'YYYYmmdd') FROM gamesmeta
        #    WHERE season = (select max(season) from seasons) AND game_date < now()::date AND  q1 IS NULL
        #    ORDER BY game_date DESC;"""
        #q = """SELECT '00' || game_id as gid, to_char(game_date, 'YYYYmmdd') as gd FROM cs_games
        #    WHERE game_date < (CURRENT_TIMESTAMP AT TIME ZONE 'CST')::date AND
        #    game_id NOT IN (SELECT DISTINCT game_id FROM boxv2015)
        #    ORDER BY game_date DESC;"""
        q = """SELECT '00' || game_id as gid, to_char(game_date, 'YYYYmmdd') as gd FROM games
            WHERE game_date < localdate() AND
            season > 2015 AND game_id NOT IN (SELECT DISTINCT game_id FROM boxv2015)
            ORDER BY game_date DESC;"""

        for g in self.db.select_dict(q):
            try:
                content = self.scraper.boxscore_v2015(g['gid'], g['gd'])
                v, h = self.parser.boxscore_v2015(content)
                self.db.insert_dicts([v, h], 'boxv2015')
                logging.info('finished {} - {}'.format(g['gd'], g['gid']))
            except Exception as e:
                logging.error('could not get {}'.format(g))
                logging.exception(e)

    def new_players(self, season):
        '''
        Updates players table with missing players

        Arguments:
            season (str): in YYYY-YY format

        Returns:
            list of players to add to stats.players

        Examples:
            a = NBAComAgent(cache_name='newplayers', cookies=httplib.CookieJar(), db=NBAComPg(...))
            np = a.new_players(season='2015-16')
        '''
        content = self.scraper.players_v2015(season)
        players = self.parser.players_v2015(content)
        currids = set([int(p.get('personId', 0)) for p in players])
        logging.debug(currids)
        allids = set(
            self.db.select_list('SELECT nbacom_player_id from players'))
        missing = currids - allids
        if missing:
            np = [p for p in players if int(p['personId']) in missing]
            if self.insert_db:
                self.db.insert_players_v2015(np)
            return np
        else:
            return None

    def player_boxscores_combined(self):
        '''
        Fetches player boxscores combined

        Arguments:
            season: str in YYYY-YY format (2015-16)

        Returns:
             players (list): player boxscores
        '''
        pboxes = []
        gids = self.db.select_list(missing_player_boxscores())
        if not gids:
            logging.error('no missing gameids found')
            return None
        logging.info('there are {} missing game boxscores'.format(len(gids)))
        for gid in gids:
            logging.info('getting {}'.format(gid))
            box = self._combined_player_boxscores(gid)
            if not box:
                logging.error('no box for {}'.format(gid))
                continue
            if self.insert_db:
                self.db.insert_player_boxscores(box)
                pboxes.append(box)
        return [item for sublist in pboxes for item in sublist]

    def player_boxscores_combined(self):
        '''
        Fetches player boxscores combined

        Arguments:
            season: str in YYYY-YY format (2015-16)

        Returns:
             players (list): player boxscores
        '''
        pboxes = []
        gids = self.db.select_list(missing_player_boxscores())
        if not gids:
            logging.error('no missing gameids found')
            return None
        logging.info('there are {} missing game boxscores'.format(len(gids)))
        for gid in gids:
            logging.info('getting {}'.format(gid))
            box = self._combined_player_boxscores(gid)
            if not box:
                logging.error('no box for {}'.format(gid))
                continue
            if self.insert_db:
                self.db.insert_player_boxscores(box)
                pboxes.append(box)
        return [item for sublist in pboxes for item in sublist]

    def player_gamelogs(self, season, date_from=None, date_to=None):
        '''
        Fetches player_gamelogs and updates player_gamelogs table

        Arguments:
            season: str in YYYY-YY format (2015-16)
            date_from: str in YYYY-mm-dd format
            date_to: str in YYYY-mm-dd format

        Returns:
             players (list): player dictionary of stats + dfs points
        '''
        pgl = self.parser.season_gamelogs(
            self.scraper.season_gamelogs(season, 'P'), 'P')
        if self.insert_db:
            self.db.insert_player_gamelogs(pgl)
        return pgl

    def playerstats(self,
                    season,
                    date_from=None,
                    date_to=None,
                    all_missing=False):
        '''
        Fetches playerstats and updates player_gamelogs table

        Arguments:
            season: str in YYYY-YY format (2015-16)
            date_from: str in YYYY-mm-dd format
            date_to: str in YYYY-mm-dd format
            all_missing: boolean

        Returns:
             players (list): player dictionary of stats + dfs points

        Examples:
            a = NBAComAgent()
            np = a.playerstats(season='2015-16', date_from='2016-03-01', date_to='2016-03-08')
        '''
        if date_from and date_to:
            ps_base = self.parser.playerstats(
                self.scraper.playerstats(season,
                                         DateFrom=date_from,
                                         DateTo=date_to))
            ps_advanced = self.parser.playerstats(
                self.scraper.playerstats(season,
                                         DateFrom=date_from,
                                         DateTo=date_to,
                                         MeasureType='Advanced'))
            ps = [
                merge(dict(), [psb, psadv])
                for psb, psadv in zip(ps_base, ps_advanced)
            ]
            if self.insert_db:
                self.db.insert_playerstats(ps, as_of=date_to)
            return ps
        elif all_missing:
            pstats = {}
            start = datetostr(d=season_start(season), site='nba')
            for day in self.db.select_list(missing_playerstats()):
                daystr = datetostr(day, 'nba')
                ps_base = self.parser.playerstats(
                    self.scraper.playerstats(season,
                                             DateFrom=start,
                                             DateTo=daystr))
                ps_advanced = self.parser.playerstats(
                    self.scraper.playerstats(season,
                                             DateFrom=start,
                                             DateTo=daystr,
                                             MeasureType='Advanced'))
                ps = [
                    merge(dict(), [psadv, psb])
                    for psb, psadv in zip(ps_base, ps_advanced)
                ]
                pstats[daystr] = ps
                if self.insert_db:
                    self.db.insert_playerstats(ps, as_of=daystr)
                    logging.info('completed {}'.format(daystr))
            return pstats
        else:
            raise ValueError(
                'need to specify dates or set all_missing to true')

    def scoreboards(self, season_start, season_end):
        '''
        Downloads and parses range of scoreboards

        Arguments:
            season_start (str): in %Y-%m-%d format
            season_end (str): in %Y-%m-%d format

        Returns:
             scoreboards (list): scoreboard dicts

         Examples:
            a = NBAComAgent()
            sb = a.scoreboards(season_start='2015-10-27', season_end='2016-04-15')
        '''
        scoreboards = []
        for day in reversed(date_list(season_end, season_start)):
            game_date = dt.datetime.strftime(day, '%Y-%m-%d')
            scoreboard_json = self.scraper.scoreboard(game_date=game_date)
            scoreboard = self.parser.scoreboard(scoreboard_json,
                                                game_date=game_date)
            scoreboards.append(scoreboard)

        if self.insert_db:
            self.db.insert_scoreboards(scoreboards)
        return scoreboards

    def team_boxscores_combined(self):
        '''
        Fetches team boxscores combined

        Returns:
             tboxes: list of boxscores
        '''
        tboxes = []
        gids = self.db.select_list(missing_team_boxscores())
        if not gids:
            logging.error('no missing gameids found')
            return None
        logging.info('there are {} missing game boxscores'.format(len(gids)))
        for gid in gids:
            logging.info('getting {}'.format(gid))
            box = self.combined_team_boxscores(gid)
            if not box:
                logging.error('no box for {}'.format(gid))
                continue
            if self.insert_db:
                self.db.insert_team_boxscores(box)
                tboxes.append(box)
        return [item for sublist in tboxes for item in sublist]

    def team_gamelogs(self, season, date_from=None, date_to=None):
        '''
        Fetches team_gamelogs and updates cs_team_gamelogs table

        Arguments:
             season (str): in YYYY-YY format (2015-16)

        Returns:
             team_gl (list): player dictionary of stats

        Examples:
            a = NBAComAgent()
            tgl = a.team_gamelogs(season='2015-16', date_from='2016-03-01', date_to='2016-03-08', insert_db=True)

        '''
        content = self.scraper.season_gamelogs(season=season,
                                               player_or_team='T')
        tgl = self.parser.season_gamelogs(content, 'T')
        mtgl = self.db.missing_tgl()
        if tgl and mtgl:
            toins = [gl for gl in tgl if gl.get('GAME_ID', None) in mtgl]
            if self.insert_db:
                self.db.insert_team_gamelogs(toins)
            return toins
        else:
            logging.error('no team gamelogs to insert')

    def teamstats(self,
                  season,
                  date_from=None,
                  date_to=None,
                  all_missing=False):
        '''
        Fetches teamstats and updates database table

        Arguments:
             season (str): in YYYY-YY format (2015-16)
             date_from (str): in %Y-%m-%d format, default beginning of season
             date_from (str): in %Y-%m-%d format, default yesterday
             all_missing: boolean

        Returns:
             list of team dictionary of basic and advanced stats

         Examples:
            a = NBAComAgent()
            ps = a.teamstats(season='2015-16', date_from='2016-03-01', date_to='2016-03-08')
            ps = a.teamstats(season='2015-16', all_missing=True)
        '''
        if date_from and date_to:
            ts_base = self.parser.teamstats(
                self.scraper.teamstats(season,
                                       DateFrom=date_from,
                                       DateTo=date_to))
            ts_advanced = self.parser.teamstats(
                self.scraper.teamstats(season,
                                       DateFrom=date_from,
                                       DateTo=date_to,
                                       MeasureType='Advanced'))
            ts = [
                merge(dict(), [psb, psadv])
                for psb, psadv in zip(ts_base, ts_advanced)
            ]
            if self.insert_db:
                self.db.insert_teamstats(ts, as_of=date_to)
            return ts
        elif all_missing:
            tstats = {}
            start = datetostr(d=season_start(season), site='nba')
            for day in self.db.select_list(missing_teamstats()):
                daystr = datetostr(day, 'nba')
                ts_base = self.parser.teamstats(
                    self.scraper.teamstats(season,
                                           DateFrom=start,
                                           DateTo=daystr))
                ts_advanced = self.parser.teamstats(
                    self.scraper.teamstats(season,
                                           DateFrom=start,
                                           DateTo=daystr,
                                           MeasureType='Advanced'))
                ts = [
                    merge(dict(), [psb, psadv])
                    for psb, psadv in zip(ts_base, ts_advanced)
                ]
                tstats[daystr] = ts
                if self.insert_db:
                    self.db.insert_teamstats(ts, as_of=daystr)
                    logging.debug('teamstats: completed {}'.format(daystr))
                else:
                    logging.error('did not insert: {}'.format(ts))
            return tstats
        else:
            raise ValueError(
                'need to specify dates or set all_missing to true')

    def team_opponent_dashboards(self,
                                 season,
                                 date_from=None,
                                 date_to=None,
                                 all_missing=False):
        '''
        Downloads and parses range of team_opponents

        Arguments:
            season (str): in YYYY-YY format
            date_from (str): in %Y-%m-%d format, default is actual start of season
            date_to (str): in %Y-%m-%d format, default is actual end of season
            all_missing (bool): get all missing dashboards

        Returns:
             topp (list): dicts

         Examples:
            a = NBAComAgent()
            topp = a.team_opponent_dashboards('2014-15')
        '''
        if date_from and date_to:
            content = self.scraper.team_opponent_dashboard(season,
                                                           DateFrom=date_from,
                                                           DateTo=date_to)
            topp = self.parser.team_opponent_dashboard(content)
            if self.insert_db:
                self.db.insert_team_opponent_dashboards(topp, as_of=date_to)
            return topp

        elif all_missing:
            topps = {}
            start = datetostr(d=season_start(season), site='nba')
            for day in self.db.select_list(missing_team_opponent_dashboard()):
                daystr = datetostr(day, 'nba')
                content = self.scraper.team_opponent_dashboard(season,
                                                               DateFrom=start,
                                                               DateTo=daystr)
                topp = self.parser.team_opponent_dashboard(content)
                if self.insert_db:
                    self.db.insert_team_opponent_dashboards(topp, as_of=daystr)
                topps[daystr] = topp
            return topps

        else:
            raise ValueError(
                'need to specify dates or set all_missing to true')

    def update_player_positions(self):
        '''
        Trying to make sure all position data is current
        Only info in nba.com is PLAYER key, this is only Guard, etc.
        Unclear where the PG, etc. comes from
        TODO: this is not functional yet
        '''
        if not self.db:
            raise ValueError('need database connection to update players')
        q = """SELECT nbacom_player_id FROM stats.players2 WHERE nbacom_position IS NULL or nbacom_position = ''"""
        uq = """UPDATE stats.players2 SET nbacom_position = '{}' WHERE nbacom_player_id = {}"""

        for pid in self.db.select_list(q):
            logging.debug('getting {}'.format(pid))
            pinfo = self.parser.player_info(
                self.scraper.player_info(pid, '2015-16'))
            if pinfo.get('POSITION'):
                self.db.update(uq.format(pinfo.get('POSITION'), pid))
                logging.debug('inserted {}'.format(
                    pinfo.get('DISPLAY_FIRST_LAST')))
Beispiel #2
0
class NBAComScraper_test(unittest.TestCase):
    def setUp(self):
        logging.getLogger(__name__).addHandler(logging.NullHandler())
        self.nbs = NBAComScraper()

    def test_boxscore(self):
        box = self.nbs.boxscore_traditional('0021500001')
        self.assertIsInstance(box, dict)
        self.assertIsNotNone(box.get('resultSets', None))
        box = self.nbs.boxscore_advanced('0021500001')
        self.assertIsInstance(box, dict)
        self.assertIsNotNone(box.get('resultSets', None))
        box = self.nbs.boxscore_scoring('0021500001')
        self.assertIsInstance(box, dict)
        self.assertIsNotNone(box.get('resultSets', None))
        box = self.nbs.boxscore_misc('0021500001')
        self.assertIsInstance(box, dict)
        self.assertIsNotNone(box.get('resultSets', None))
        box = self.nbs.boxscore_usage('0021500001')
        self.assertIsInstance(box, dict)
        self.assertIsNotNone(box.get('resultSets', None))

    def test_playerstats(self):
        ps = self.nbs.playerstats('2016-17')
        self.assertIsInstance(ps, dict)
        rs = ps.get('resultSets', None)
        self.assertIsNotNone(rs)
        self.assertIsNotNone(rs[0].get('headers', None))

    def test_player_info(self):
        pinfo = self.nbs.player_info('203083', '2016-17')
        self.assertIsInstance(pinfo, dict)
        rs = pinfo.get('resultSets', None)
        self.assertIsNotNone(rs)
        self.assertIsNotNone(rs[0].get('headers', None))

    def test_one_player_gamelogs(self):
        pgl = self.nbs.one_player_gamelogs('203083', '2016-17')
        self.assertIsInstance(pgl, dict)
        rs = pgl.get('resultSets', None)
        self.assertIsNotNone(rs)
        self.assertIsNotNone(rs[0].get('headers', None))

    def test_players(self):
        players = self.nbs.players(season='2016-17', cs_only='1')
        self.assertIsInstance(players, dict)
        rs = players.get('resultSets', None)
        self.assertIsNotNone(rs)
        self.assertIsNotNone(rs[0].get('headers', None))

    def test_season_gamelogs(self):
        pgl = self.nbs.season_gamelogs(season='2016-17', player_or_team='P')
        self.assertIsInstance(pgl, dict)
        rs = pgl.get('resultSets', None)
        self.assertIsNotNone(rs)
        self.assertIsNotNone(rs[0].get('headers', None))
        tgl = self.nbs.season_gamelogs(season='2016-17', player_or_team='T')
        self.assertIsInstance(tgl, dict)
        rs = tgl.get('resultSets', None)
        self.assertIsNotNone(rs)
        self.assertIsNotNone(rs[0].get('headers', None))

    def test_team_dashboard(self):
        team_id = '1610612765'
        season = '2016-17'
        tdb = self.nbs.team_dashboard(team_id, season)
        self.assertIsInstance(tdb, dict)
        rs = tdb.get('resultSets', None)
        self.assertIsNotNone(rs)
        self.assertIsNotNone(rs[0].get('headers', None))

    def test_team_opponent_dashboard(self):
        season = '2016-17'
        tdb = self.nbs.team_opponent_dashboard(season)
        self.assertIsInstance(tdb, dict)
        rs = tdb.get('resultSets', None)
        self.assertIsNotNone(rs)
        self.assertIsNotNone(rs[0].get('headers', None))

    def test_one_team_gamelogs(self):
        team_id = '1610612765'
        season = '2016-17'
        tdb = self.nbs.one_team_gamelogs(team_id, season)
        self.assertIsInstance(tdb, dict)
        rs = tdb.get('resultSets', None)
        self.assertIsNotNone(rs)
        self.assertIsNotNone(rs[0].get('headers', None))

    def test_teamstats(self):
        season = '2016-17'
        ts = self.nbs.team_opponent_dashboard(season)
        self.assertIsInstance(ts, dict)
        rs = ts.get('resultSets', None)
        self.assertIsNotNone(rs)
        self.assertIsNotNone(rs[0].get('headers', None))

    def test_teams(self):
        t = self.nbs.teams()
        self.assertIsInstance(t, basestring)
        self.assertRegexpMatches(t, r'Pistons')
Beispiel #3
0
class NBAComAgent(object):
    '''
    Performs script-like tasks using NBA.com API
    '''
    def __init__(self,
                 db=None,
                 cache_name='nbacom-agent',
                 cookies=None,
                 table_names=None):
        '''
        Args:
            db (NBAPostgres): instance
            cache_name (str): for scraper cache_name
            cookies: cookie jar
            table_names (dict): Database table names

        '''
        logging.getLogger(__name__).addHandler(logging.NullHandler())
        self.scraper = NBAComScraper(cache_name=cache_name, cookies=cookies)
        self.parser = NBAComParser()
        self.db = db
        if table_names:
            self.table_names = table_names
        else:
            self.table_names = {
                'pgl': 'player_gamelogs',
                'tgl': 'team_gamelogs',
                'seas': 'season',
                'pl': 'player',
                'ps': 'playerstats_daily',
                'ts': 'teamstats_daily',
                'tod': 'team_opponent_dashboard',
                'pbs': 'player_boxscores_combined',
                'tbs': 'team_boxscores_combined',
                'box2': 'game_boxscores',
                'tm': 'team'
            }

    def combined_boxscores(self, game_ids=None):
        '''
            Combines 5 types of boxscores (traditional, advanced, misc, scoring, usage) into list of team boxscore

            Arguments:
                game_id (str): game ID, with leading '00'

            Returns:
                tuple: list of player boxscores, list of team boxscores

            '''
        player_boxscores = []
        team_boxscores = []
        if not game_ids:
            game_ids = set(self.db.select_list(missing_player_boxscores())) | \
                   set(self.db.select_list(missing_team_boxscores()))

        # need union of missing player and team
        for game_id in game_ids:
            # make sure game_id in correct format when requesting boxscores
            gid = nbacom_idstr(game_id)
            logging.info(
                'agent.nbacom.combined_boxscores: getting {}'.format(gid))

            # get 5 types of boxscores
            traditional_players, traditional_teams, traditional_starter_bench = \
                self.parser.boxscore_traditional(self.scraper.boxscore_traditional(gid))
            adv_players, adv_teams = self.parser.boxscore_advanced(
                self.scraper.boxscore_advanced(gid))
            misc_players, misc_teams = self.parser.boxscore_misc(
                self.scraper.boxscore_misc(gid))
            scoring_players, scoring_teams = self.parser.boxscore_scoring(
                self.scraper.boxscore_scoring(gid))
            usage_players = self.parser.boxscore_usage(
                self.scraper.boxscore_usage(gid))

            # combine player and team boxscores, respectively
            players_combined = list(
                merge_many(dict(),
                           [{t['PLAYER_ID']: t
                             for t in traditional_players},
                            {t['PLAYER_ID']: t
                             for t in adv_players},
                            {t['PLAYER_ID']: t
                             for t in misc_players},
                            {t['PLAYER_ID']: t
                             for t in scoring_players},
                            {t['PLAYER_ID']: t
                             for t in usage_players}]).values())
            teams_combined = list(
                merge_many(dict(), [{
                    t['TEAM_ID']: t
                    for t in traditional_teams
                }, {t['TEAM_ID']: t
                    for t in adv_teams}, {t['TEAM_ID']: t
                                          for t in misc_teams},
                                    {t['TEAM_ID']: t
                                     for t in scoring_teams}]).values())

            # now add to the database
            self.db.safe_insert_dicts(player_boxscores_table(players_combined),
                                      self.table_names['pbs'])
            self.db.safe_insert_dicts(team_boxscores_table(teams_combined),
                                      self.table_names['tbs'])
        return (player_boxscores, team_boxscores)

    def game_boxscores(self, game_ids=None):
        '''
        Updates table with game_information
        
        Args:
            None
            
        Returns:
            None
        '''
        if not game_ids:
            game_ids = self.db.select_dict(missing_game_boxscores())

        for g in game_ids:
            try:
                content = self.scraper.boxscore_v2015(g['gid'], g['gd'])
                v, h = self.parser.boxscore_v2015(content)
                self.db._insert_dict(v, self.table_names['box2'])
                self.db._insert_dict(h, self.table_names['box2'])
                logging.info('finished {} - {}'.format(g['gd'], g['gid']))
                return [v, h]
            except Exception as e:
                logging.error('could not get {}'.format(g))
                logging.exception(e)
                return None

    def gleague_players(self, year):
        '''
        Updates player table with g-league players

        Args:
            year (int): 2017, etc.

        Returns:
            None

        '''
        content = self.scraper.gleague_players(year)
        players = self.parser.gleague_players(content)
        for glp in gleague_player_table(players):
            self.db._insert_dict(glp, self.table_names['pl'])
        return players

    def player_gamelogs(self, season_code, date_from=None, date_to=None):
        '''
        Fetches player_gamelogs and updates player_gamelogs table

        Args:
            season_code (str): in YYYY-YY format (2017-18)
            date_from (str): in YYYY-mm-dd format, default None
            date_to (str): in YYYY-mm-dd format, default None

        Returns:
             list: of player dict

        '''
        # get all player gamelogs from nba.com
        content = self.scraper.season_gamelogs(season_code,
                                               'P',
                                               date_from=date_from,
                                               date_to=date_to)
        pgl = self.parser.season_gamelogs(content, 'P')
        pgl_s = set(
            ['{}-{}'.format(gl['GAME_ID'], gl['PLAYER_ID']) for gl in pgl])

        # compare to gamelogs in database: refresh view then compare
        dbpgl = self.db.select_dict(
            'SELECT nbacom_game_id, nbacom_player_id FROM cs_player_gamelogs')
        dbpgl_s = set([
            '00{}-{}'.format(gl['nbacom_game_id'], gl['nbacom_player_id'])
            for gl in dbpgl
        ])

        # only try to insert missing gamelogs
        missing = pgl_s - dbpgl_s
        to_ins = [
            gl for gl in pgl
            if '{}-{}'.format(gl['GAME_ID'], gl['PLAYER_ID']) in missing
        ]
        self.db.insert_dicts(player_gamelogs_table(to_ins), 'player_gamelogs')
        return pgl

    def playerstats(self,
                    season_code,
                    per_mode='Totals',
                    date_from=None,
                    date_to=None,
                    all_missing=False):
        '''
        Fetches playerstats and updates playerstats table

        Args:
            season_code (str): in YYYY-YY format, e.g. 2017-18
            per_mode (str): 'Totals', 'PerGame', etc.
            date_from (str): in YYYY-mm-dd format, default None
            date_to (str): in YYYY-mm-dd format, default None
            all_missing (bool): default False

        Returns:
             None

        '''
        if date_from and date_to:
            ps_base = self.parser.playerstats(
                self.scraper.playerstats(season_code,
                                         per_mode=per_mode,
                                         DateFrom=date_from,
                                         DateTo=date_to))
            ps_advanced = self.parser.playerstats(
                self.scraper.playerstats(season_code,
                                         per_mode=per_mode,
                                         DateFrom=date_from,
                                         DateTo=date_to,
                                         MeasureType='Advanced'))
            ps = [
                merge_two(psb, psadv)
                for psb, psadv in zip(ps_base, ps_advanced)
            ]
            vals = playerstats_table(ps, date_to, per_mode)
            logging.info(vals)
            self.db.insert_dicts(vals, self.table_names['ps'])
        elif all_missing:
            start = datetostr(d=season_start(season_code=season_code),
                              fmt='nba')
            for day in self.db.select_list(missing_playerstats(per_mode)):
                daystr = datetostr(day, 'nba')
                base_content = self.scraper.playerstats(season_code,
                                                        per_mode=per_mode,
                                                        DateFrom=start,
                                                        DateTo=daystr)
                ps_base = self.parser.playerstats(base_content,
                                                  per_mode=per_mode)
                adv_content = self.scraper.playerstats(season_code,
                                                       per_mode=per_mode,
                                                       DateFrom=start,
                                                       DateTo=daystr,
                                                       MeasureType='Advanced')
                ps_advanced = self.parser.playerstats(adv_content, per_mode)
                ps = [
                    merge_two(psadv, psb)
                    for psb, psadv in zip(ps_base, ps_advanced)
                ]
                self.db.insert_dicts(playerstats_table(ps, daystr, per_mode),
                                     self.table_names['ps'])
                logging.info('completed {}'.format(daystr))
        else:
            raise ValueError(
                'agent.nbacom.playerstats: need to specify dates or set all_missing to true'
            )

    def refresh_materialized(self):
        '''
        Calls postgres function to refresh all materialized views.
        
        '''
        refreshq = """SELECT RefreshAllMaterializedViews('*', true);"""
        try:
            self.db.execute(refreshq)
        except Exception as e:
            logging.exception(
                'could not refresh materialized views: {}'.format(e))

    def team_gamelogs(self, season_code, date_from=None, date_to=None):
        '''
        Fetches team_gamelogs and updates team_gamelogs table

        Args:
            season_code (str): in YYYY-YY format (2017-18)
            date_from (str): in YYYY-mm-dd format, default None
            date_to (str): in YYYY-mm-dd format, default None
            
        Returns:
            list: of dict

        '''
        content = self.scraper.season_gamelogs(season_code, 'T', date_from,
                                               date_to)
        tgl = self.parser.season_gamelogs(content, 'T')
        tgl_s = set(
            ['{}-{}'.format(gl['GAME_ID'], gl['TEAM_ID']) for gl in tgl])

        # compare team gamelogs to those already in database
        dbtgl_s = set(
            self.db.select_list(
                """SELECT CONCAT(nbacom_game_id, '-', nbacom_team_id) FROM cs_team_gamelogs"""
            ))

        # only try to insert missing gamelogs
        missing = tgl_s - dbtgl_s
        to_ins = [
            gl for gl in tgl
            if '{}-{}'.format(gl['GAME_ID'], gl['TEAM_ID']) in missing
        ]
        for item in team_gamelogs_table(to_ins):
            self.db._insert_dict(item, self.table_names['tgl'])
        return tgl

    def team_opponent_dashboards(self,
                                 season_code,
                                 per_mode,
                                 date_from=None,
                                 date_to=None,
                                 all_missing=False):
        '''
        Downloads and parses range of team_opponents

        Arguments:
            season_code (str): in YYYY-YY format
            per_mode (str): 'Totals', 'PerGame', etc.
            date_from (str): in %Y-%m-%d format, default is actual start of season
            date_to (str): in %Y-%m-%d format, default is actual end of season
            all_missing (bool): get all missing dashboards

        Returns:
            None
            
        '''
        if date_from and date_to:
            content = self.scraper.team_opponent_dashboard(season_code,
                                                           per_mode=per_mode,
                                                           date_from=date_from,
                                                           date_to=date_to)
            topp = self.parser.team_opponent_dashboard(content)
            self.db.insert_dicts(team_opponent_dashboards_table(topp, date_to),
                                 self.table_names.get('tod'))
        elif all_missing:
            topps = {}
            start = datetostr(d=season_start(season_code=season_code),
                              fmt='nba')
            for day in self.db.select_list(
                    missing_team_opponent_dashboard(per_mode)):
                daystr = datetostr(day, 'nba')
                logging.info('starting dashboards for {}'.format(daystr))
                content = self.scraper.team_opponent_dashboard(season_code,
                                                               date_from=start,
                                                               date_to=date_to)
                topp = self.parser.team_opponent_dashboard(content, per_mode)
                self.db.insert_dicts(
                    team_opponent_dashboards_table(topp, daystr, per_mode),
                    self.table_names.get('tod'))
            return topps
        else:
            raise ValueError(
                'need to specify dates or set all_missing to true')

    def teamstats(self,
                  season_code,
                  per_mode,
                  date_from=None,
                  date_to=None,
                  all_missing=False):
        '''
        Fetches teamstats and updates database table

        Args:
             season_code (str): in YYYY-YY format (2015-16)
             per_mode (str): 'Totals', 'PerGame', etc.
             date_from (str): in %Y-%m-%d format
             date_to (str): in %Y-%m-%d format
             all_missing (bool): looks for all missing teamstats from season

        Returns:
             list of team dictionary of basic and advanced stats

        '''
        if date_from and date_to:
            content_base = self.scraper.teamstats(season_code,
                                                  per_mode=per_mode,
                                                  date_from=date_from,
                                                  date_to=date_to)
            ts_base = self.parser.teamstats(content_base, per_mode)
            content_adv = self.scraper.teamstats(season_code,
                                                 per_mode=per_mode,
                                                 date_from=date_from,
                                                 date_to=date_to,
                                                 MeasureType='Advanced')
            ts_advanced = self.parser.teamstats(content_adv, per_mode)
            ts_merged = [
                merge_two(tsb, tsadv)
                for tsb, tsadv in zip(ts_base, ts_advanced)
            ]
            self.db.insert_dicts(teamstats_table(ts_merged, date_to, per_mode),
                                 self.table_names['ts'])
        elif all_missing:
            start = datetostr(d=season_start(season_code=season_code),
                              fmt='nba')
            for day in self.db.select_list(missing_teamstats(per_mode)):
                logging.info('teamstats: getting {}'.format(day))
                daystr = datetostr(day, 'nba')
                content_base = self.scraper.teamstats(season_code,
                                                      per_mode=per_mode,
                                                      date_from=start,
                                                      date_to=daystr)
                ts_base = self.parser.teamstats(content_base, per_mode)
                content_adv = self.scraper.teamstats(season_code,
                                                     per_mode=per_mode,
                                                     date_from=start,
                                                     date_to=daystr,
                                                     MeasureType='Advanced')
                ts_advanced = self.parser.teamstats(content_adv, per_mode)
                ts = [
                    merge_two(tsb, tsadv)
                    for tsb, tsadv in zip(ts_base, ts_advanced)
                ]
                self.db.insert_dicts(teamstats_table(ts, daystr, per_mode),
                                     self.table_names['ts'])
                logging.debug('teamstats: completed {}'.format(daystr))
        else:
            raise ValueError(
                'need to specify dates or set all_missing to true')
Beispiel #4
0
class NBAComAgent(object):
    '''
    Performs script-like tasks using NBA.com API
    '''
    def __init__(self, db=None, cache_name=None, cookies=None):
        '''
        Arguments:
            cache_name: str for scraper cache_name
            cookies: cookie jar
            db: NBAComPg instance
        '''
        self.scraper = NBAComScraper(cache_name=cache_name, cookies=cookies)
        self.parser = NBAComParser()
        if db:
            self.db = db
            self.insert_db = True
        else:
            self.insert_db = False

    def _merge(self, merge_dico, dico_list):
        '''
        See http://stackoverflow.com/questions/28838291/merging-multiple-dictionaries-in-python
        Args:
            merge_dico:
            dico_list:
        Returns:
            merged dictionary
        '''
        for dico in dico_list:
            for key, value in dico.items():
                if type(value) == type(dict()):
                    merge_dico.setdefault(key, dict())
                    self._merge(merge_dico[key], [value])
                else:
                    merge_dico[key] = value
        return merge_dico

    def combined_player_boxscores(self, gid):
        '''
        Combines 5 types of boxscores from nba.com API into list of boxscores
        Arguments:
            gid: string game ID, with leading '00'
        Returns:
            list of player boxscores combined - traditional, advanced, misc, scoring, usage
        Examples:
            a = NBAComAgent()
            playerboxes = a.combined_player_boxscores('0020161001')
        '''
        traditional_players, traditional_teams, traditional_starter_bench = self.parser.boxscore_traditional(
            self.scraper.boxscore_traditional(gid))
        adv_players, adv_team = self.parser.boxscore_advanced(
            self.scraper.boxscore_advanced(gid))
        misc_players, misc_team = self.parser.boxscore_misc(
            self.scraper.boxscore_misc(gid))
        scoring_players, scoring_team = self.parser.boxscore_scoring(
            self.scraper.boxscore_scoring(gid))
        usage_players = self.parser.boxscore_usage(
            self.scraper.boxscore_usage(gid))

        # now need to combine player and team boxscores
        players = self._merge(
            dict(),
            [{t['PLAYER_ID']: t
              for t in traditional_players},
             {t['PLAYER_ID']: t
              for t in adv_players}, {t['PLAYER_ID']: t
                                      for t in misc_players},
             {t['PLAYER_ID']: t
              for t in scoring_players},
             {t['PLAYER_ID']: t
              for t in usage_players}])
        return list(players.values())

    def combined_team_boxscores(self, gid):
        '''
        Combines 5 types of boxscores from nba.com API into list of boxscores
        Arguments:
            gid: string game ID, with leading '00'
        Returns:
            list of team boxscores - combined traditional, advanced, misc, scoring
        Examples:
            a = NBAComAgent()
            teamboxes = a.combined_team_boxscores('0020161001')
        '''
        traditional_players, traditional_teams, traditional_starter_bench = self.parser.boxscore_traditional(
            self.scraper.boxscore_traditional(gid))
        adv_players, adv_teams = self.parser.boxscore_advanced(
            self.scraper.boxscore_advanced(gid))
        misc_players, misc_teams = self.parser.boxscore_misc(
            self.scraper.boxscore_misc(gid))
        scoring_players, scoring_teams = self.parser.boxscore_scoring(
            self.scraper.boxscore_scoring(gid))

        # now need to combine player and team boxscores
        teams = self._merge(
            dict(),
            [{t['TEAM_ID']: t
              for t in traditional_teams},
             {t['TEAM_ID']: t
              for t in adv_teams}, {t['TEAM_ID']: t
                                    for t in misc_teams},
             {t['TEAM_ID']: t
              for t in scoring_teams}])
        return list(teams.values())

    def new_players(self, season):
        '''
        Updates players table with missing players

        Arguments:
            season (str): in YYYY-YY format
        Returns:
            list of players to add to stats.players
        Examples:
            a = NBAComAgent(cache_name='newplayers', cookies=httplib.CookieJar(), db=NBAComPg(...))
            np = a.new_players(season='2015-16')
        '''
        content = self.scraper.players(season, cs_only=1)
        players = self.parser.players(content)
        currids = set([int(p.get('PERSON_ID', 0)) for p in players])
        allids = set(
            self.db.select_list('SELECT nbacom_player_id from players'))
        missing = currids - allids
        if missing:
            np = [
                self.parser.player_info(self.scraper.player_info(pid, season))
                for pid in missing
            ]
            if self.insert_db:
                self.db.insert_players(np)
            return np
        else:
            return None

    def player_gamelogs(self, season, date_from=None, date_to=None):
        '''
        Fetches player_gamelogs and updates player_gamelogs table

        Arguments:
            season: str in YYYY-YY format (2015-16)
            date_from: str in YYYY-mm-dd format
            date_to: str in YYYY-mm-dd format
            insert_db: add list to database

        Returns:
             players (list): player dictionary of stats + dfs points

        Examples:
            a = NBAComAgent()
            np = a.player_gamelogs(season='2015-16', date_from='2016-03-01', date_to='2016-03-08', insert_db=True)
        '''
        # mpgl -> list of game_ids for which there are no player gamelogs
        # filter gamelogs to those from missing game_ids
        pgl = self.parser.season_gamelogs(
            self.scraper.season_gamelogs(season, 'P'), 'P')
        mpgl = self.db.missing_pgl()
        if pgl and mpgl:
            pgl = [gl for gl in pgl if gl.get('GAME_ID', None) in mpgl]
            # now make sure you have no new players
            currids = set([int(p.get('PERSON_ID', 0)) for p in pgl])
            allids = set(
                self.db.select_list('SELECT nbacom_player_id from players'))
            if self.insert_db:
                self.db.insert_players([
                    self.parser.player_info(
                        self.scraper.player_info(pid, season))
                    for pid in currids - allids
                ])
                self.db.insert_player_gamelogs(pgl)
            return pgl
        else:
            return None

    def playerstats(self,
                    season,
                    date_from=None,
                    date_to=None,
                    all_missing=False):
        '''
        Fetches playerstats and updates player_gamelogs table

        Arguments:
            season: str in YYYY-YY format (2015-16)
            date_from: str in YYYY-mm-dd format
            date_to: str in YYYY-mm-dd format
            all_missing: boolean

        Returns:
             players (list): player dictionary of stats + dfs points

        Examples:
            a = NBAComAgent()
            np = a.playerstats(season='2015-16', date_from='2016-03-01', date_to='2016-03-08')
        '''
        if date_from and date_to:
            ps_base = self.parser.playerstats(
                self.scraper.playerstats(season,
                                         DateFrom=date_from,
                                         DateTo=date_to))
            ps_advanced = self.parser.playerstats(
                self.scraper.playerstats(season,
                                         DateFrom=date_from,
                                         DateTo=date_to,
                                         MeasureType='Advanced'))
            ps = [
                self._merge(dict(), [psb, psadv])
                for psb, psadv in zip(ps_base, ps_advanced)
            ]
            if self.insert_db:
                self.db.insert_playerstats(ps, as_of=date_to)
            return ps
        elif all_missing:
            pstats = {}
            start = datetostr(d=season_start(season), site='nba')
            for day in self.db.select_list(
                    'SELECT game_date FROM missing_playerstats'):
                daystr = datetostr(day, 'nba')
                ps_base = self.parser.playerstats(
                    self.scraper.playerstats(season,
                                             DateFrom=start,
                                             DateTo=daystr))
                ps_advanced = self.parser.playerstats(
                    self.scraper.playerstats(season,
                                             DateFrom=start,
                                             DateTo=daystr,
                                             MeasureType='Advanced'))
                ps = [
                    self._merge(dict(), [psb, psadv])
                    for psb, psadv in zip(ps_base, ps_advanced)
                ]
                pstats[daystr] = ps
                if self.insert_db:
                    self.db.insert_playerstats(ps, as_of=daystr)
            return pstats
        else:
            raise ValueError(
                'need to specify dates or set all_missing to true')

    def scoreboards(self, season_start, season_end):
        '''
        Downloads and parses range of scoreboards

        Arguments:
            season_start (str): in %Y-%m-%d format
            season_end (str): in %Y-%m-%d format

        Returns:
             scoreboards (list): scoreboard dicts

         Examples:
            a = NBAComAgent()
            sb = a.scoreboards(season_start='2015-10-27', season_end='2016-04-15')
        '''
        scoreboards = []
        for day in reversed(date_list(season_end, season_start)):
            game_date = dt.datetime.strftime(day, '%Y-%m-%d')
            scoreboard_json = self.scraper.scoreboard(game_date=game_date)
            scoreboard = self.parser.scoreboard(scoreboard_json,
                                                game_date=game_date)
            scoreboards.append(scoreboard)

        if self.insert_db:
            self.db.insert_scoreboards(scoreboards)
        return scoreboards

    def team_gamelogs(self, season, date_from=None, date_to=None):
        '''
        Fetches team_gamelogs and updates cs_team_gamelogs table

        Arguments:
             season (str): in YYYY-YY format (2015-16)

        Returns:
             team_gl (list): player dictionary of stats

        Examples:
            a = NBAComAgent()
            tgl = a.team_gamelogs(season='2015-16', date_from='2016-03-01', date_to='2016-03-08', insert_db=True)

        '''
        content = self.scraper.season_gamelogs(season=season,
                                               player_or_team='T')
        tgl = self.parser.season_gamelogs(content, 'T')
        mtgl = self.db.missing_tgl()
        if tgl and mtgl:
            toins = [gl for gl in tgl if gl.get('GAME_ID', None) in mtgl]
            if self.insert_db:
                self.db.insert_team_gamelogs(toins)
            return toins
        else:
            logging.error('no team gamelogs to insert')

    def teamstats(self,
                  season,
                  date_from=None,
                  date_to=None,
                  all_missing=False):
        '''
        Fetches teamstats and updates database table

        Arguments:
             season (str): in YYYY-YY format (2015-16)
             date_from (str): in %Y-%m-%d format, default beginning of season
             date_from (str): in %Y-%m-%d format, default yesterday
             all_missing: boolean

        Returns:
             list of team dictionary of basic and advanced stats

         Examples:
            a = NBAComAgent()
            ps = a.teamstats(season='2015-16', date_from='2016-03-01', date_to='2016-03-08')
            ps = a.teamstats(season='2015-16', all_missing=True)
        '''
        if date_from and date_to:
            ts_base = self.parser.teamstats(
                self.scraper.teamstats(season,
                                       DateFrom=date_from,
                                       DateTo=date_to))
            ts_advanced = self.parser.teamstats(
                self.scraper.teamstats(season,
                                       DateFrom=date_from,
                                       DateTo=date_to,
                                       MeasureType='Advanced'))
            ts = [
                self._merge(dict(), [psb, psadv])
                for psb, psadv in zip(ts_base, ts_advanced)
            ]
            if self.insert_db:
                self.db.insert_teamstats(ts, as_of=date_to)
            return ts
        elif all_missing:
            tstats = {}
            start = datetostr(d=season_start(season), site='nba')
            for day in self.db.select_list(
                    'SELECT game_date FROM missing_teamstats'):
                daystr = datetostr(day, 'nba')
                ts_base = self.parser.teamstats(
                    self.scraper.teamstats(season,
                                           DateFrom=start,
                                           DateTo=daystr))
                ts_advanced = self.parser.teamstats(
                    self.scraper.teamstats(season,
                                           DateFrom=start,
                                           DateTo=daystr,
                                           MeasureType='Advanced'))
                ts = [
                    self._merge(dict(), [psb, psadv])
                    for psb, psadv in zip(ts_base, ts_advanced)
                ]
                tstats[daystr] = ts
                if self.insert_db:
                    self.db.insert_teamstats(ts, as_of=daystr)
                    logging.info('teamstats: completed {}'.format(daystr))
                else:
                    logging.error('did not insert: {}'.format(ts))
            return tstats
        else:
            raise ValueError(
                'need to specify dates or set all_missing to true')

    def team_opponent_dashboards(self,
                                 season,
                                 date_from=None,
                                 date_to=None,
                                 all_missing=False):
        '''
        Downloads and parses range of team_opponents

        Arguments:
            season (str): in YYYY-YY format
            date_from (str): in %Y-%m-%d format, default is actual start of season
            date_to (str): in %Y-%m-%d format, default is actual end of season
            all_missing (bool): get all missing dashboards

        Returns:
             topp (list): dicts

         Examples:
            a = NBAComAgent()
            topp = a.team_opponent_dashboards('2014-15')
        '''
        if date_from and date_to:
            content = self.scraper.team_opponent_dashboard(season,
                                                           DateFrom=date_from,
                                                           DateTo=date_to)
            topp = self.parser.team_opponent_dashboard(content)
            if self.insert_db:
                self.db.insert_team_opponent_dashboards(topp, as_of=date_to)
            return topp

        elif all_missing:
            topps = {}
            start = datetostr(d=season_start(season), site='nba')
            for day in self.db.select_list(
                    'SELECT * FROM missing_team_opponent_dashboard'):
                daystr = datetostr(day, 'nba')
                content = self.scraper.team_opponent_dashboard(season,
                                                               DateFrom=start,
                                                               DateTo=daystr)
                topp = self.parser.team_opponent_dashboard(content)
                if self.insert_db:
                    self.db.insert_team_opponent_dashboards(topp, as_of=daystr)
                topps[daystr] = topp
            return topps

        else:
            raise ValueError(
                'need to specify dates or set all_missing to true')

    def update_player_positions(self):
        '''
        Trying to make sure all position data is current
        Only info in nba.com is PLAYER key, this is only Guard, etc.
        Unclear where the PG, etc. comes from
        TODO: this is not functional yet
        '''
        if not self.db:
            raise ValueError('need database connection to update players')
        q = """SELECT nbacom_player_id FROM stats.players2 WHERE nbacom_position IS NULL or nbacom_position = ''"""
        uq = """UPDATE stats.players2 SET nbacom_position = '{}' WHERE nbacom_player_id = {}"""

        for pid in self.db.select_list(q):
            logging.info('getting {}'.format(pid))
            pinfo = self.parser.player_info(
                self.scraper.player_info(pid, '2015-16'))
            if pinfo.get('POSITION'):
                self.db.update(uq.format(pinfo.get('POSITION'), pid))
                logging.info('inserted {}'.format(
                    pinfo.get('DISPLAY_FIRST_LAST')))