def getSalariesForDatesAndPostToApi(dateArr, site): ''' :param dateArr: array of YYYY-MM-DD date strings :param site: 'fanduel' or 'draftkings' :return: all new salaries & new ids in dict ''' driver = webdriver.PhantomJS() driver.set_window_size(1124, 850) playerData = api.getCurrentPlayerData() allData = {'allSalaries': [], 'allNewIds': []} # for each date in range: for date in dateArr: # get salary & pos data salariesForDate = fc.getSalaryDataForDate(date, site, playerData, driver) # add salary & missing player data allData['allSalaries'].extend(salariesForDate['currentPlayerSalaries']) allData['allNewIds'].extend(salariesForDate['missingPlayerIds']) newSalariesResponse = api.postSalaries(allData['allSalaries']) # UNCOMMENT LINE BELOW IF ALSO WANT TO ADD NEW FC IDS # dedupedNewIds = [i for n, i in enumerate(allData["allNewIds"]) if i not in allData["allNewIds"][n + 1:]] # newIdsResponse = api.postNewIds(dedupedNewIds) return allData
def getSourceIdUpdatesForPlayersAuto(): ''' Will auto update all players w/ source ids that have an exact name match NOTE: will override all existing source ids for any player that matches ''' # get new source ids newSourceIdsByPlayer = api.getNewSourceIds() # get list of current players currentPlayers = api.getCurrentPlayerData() # arr to hold update sourceIdUpdates = {} for idsByPlayer in newSourceIdsByPlayer: currentPlayerMatch = next( (player for player in currentPlayers if player["player_name"] == idsByPlayer["player_name"]), None) # if name does not EXACTLY match any existing player: if currentPlayerMatch is None: continue # if there is a match: else: sourceIds = mapNewSourceIdsToExistingPlayer( idsByPlayer, currentPlayerMatch) # add ids to player dict sourceIdUpdates[currentPlayerMatch["player_id"]] = sourceIds return sourceIdUpdates
def getPostGameDataForGameArr(game_arr): all_post_game_data = [] player_data = api.getCurrentPlayerData() session = requests.Session() for game in game_arr: post_game_data = post.getBoxScoreData(game["bref_slug"], session, player_data) post_game_data["gameId"] = game["game_id"] all_post_game_data.append(post_game_data) return all_post_game_data
def getDepthChartData(sessionObj): ''' Calls rw depth scraper fcns Parses data into dict based on new/current players Returns dict ''' # import team ids as dict teamDict = jsonData.TEAM_ABBREV_TO_ID # get currPlayerData = api.getCurrentPlayerData() # scrape current depth chart data & starting lineup data from rw htmlTree = rw.getRawDepthChartTree(sessionObj, config) currentDepths = rw.getPlayerDataFromTree(htmlTree, teamDict) playerIdsOnRosters = [] playerDataUpdates = {"rwIdInDbUpdates": [], "rwIdNotInDbPosts": []} # for each player in current depths, check if they are in DB: for rwId, rwData in currentDepths.items(): try: matchedPlayer = next( (player for player in currPlayerData if player["rw_id"] == rwId), None) rwData["playerId"] = matchedPlayer["player_id"] # if player is already incomplete in DB, don't override w new status if matchedPlayer["status"] == 'INCOMPLETE': rwData["status"] = 'INCOMPLETE' # add player id to arr for later check playerIdsOnRosters.append(matchedPlayer["player_id"]) # only update data if player data has changed to save api calls/ db writes if hasPlayerDataChanged(matchedPlayer, rwData): playerDataUpdates["rwIdInDbUpdates"].append(rwData) except: # if no match for rw_id, add player to rwNotInDbArr rwData["rw_id"] = rwId playerDataUpdates["rwIdNotInDbPosts"].append(rwData) # create arr of playerIds not on a roster playerDataUpdates["playersNotOnRoster"] = [ player["player_id"] for player in currPlayerData if player["player_id"] not in playerIdsOnRosters and player["status"] != 'NOT_ON_ROSTER' ] return playerDataUpdates
def getPostGameDataForYesterdaysGames(): games = getYesterdaysGames() playerData = api.getCurrentPlayerData() session = requests.Session() allPostGameData = [] for game in games: print(game["bref_slug"]) postGameData = post.getBoxScoreData(game["bref_slug"], session, playerData) allPostGameData.append(postGameData) return allPostGameData
def parseProjsFromHtml(htmlDict): # get list of curr players from API playerList = api.getCurrentPlayerData() gamesToday = api.getTodaysGames() parsedProjs = [] newIds = [] try: # get all projection data from all raw html nfData = nf.extractProjections(htmlDict["nf"], playerList, gamesToday) rwData = rw.extractProjections(htmlDict["rw"], playerList, gamesToday) fpData = fp.extractProjections(htmlDict["fp"], playerList, gamesToday) bmData = bm.extractProjections(htmlDict["bm"], playerList, gamesToday) # parsedProjs = bmData["projections"] # newIds = bmData["newPlayerIds"] # concat all projs & missing players into single arrays parsedProjs = nfData["projections"] + rwData["projections"] + fpData["projections"] + bmData["projections"] newIds = nfData["newPlayerIds"] + rwData["newPlayerIds"] + fpData["newPlayerIds"] + bmData["newPlayerIds"] counts = { "nf": { "projs": len(nfData["projections"]), "new_ids": len(nfData["newPlayerIds"]), "total_rows": nfData["totalNumRows"] }, "rw": { "projs": len(rwData["projections"]), "new_ids": len(rwData["newPlayerIds"]), "total_rows": rwData["totalNumRows"] }, "fp": { "projs": len(fpData["projections"]), "new_ids": len(fpData["newPlayerIds"]), "total_rows": fpData["totalNumRows"] }, "bm": { "projs": len(bmData["projections"]), "new_ids": len(bmData["newPlayerIds"]), "total_rows": bmData["totalNumRows"] }, } except Exception as error: # TODO: handle this err. Log? Throw? print("COULDN'T PARSE PROJECTIONS, ERROR:", error) return {'projections': parsedProjs, 'newPlayerIds': newIds, 'counts': counts}
def scrapeGameStats(gameOrMinDate, maxDate=None): if maxDate == None: gamesToScrape = api.getGamesForDate(gameOrMinDate) else: gamesToScrape = api.getGamesInRange(gameOrMinDate, maxDate) playerList = api.getCurrentPlayerData() session = requests.Session() allGameStats = [] for game in gamesToScrape: statsForGame = actual.getGameStats(game, playerList, session) allGameStats.extend(statsForGame) return allGameStats
def get_usual_depth_positions(season): session = requests.Session() current_players = api.getCurrentPlayerData() # teams_url = 'http://www.basketball-reference.com/leagues/NBA_' + str(season) + '.html' depth_url = 'http://basketball.realgm.com/nba/depth-charts/' + str(season) page = session.get(depth_url) tree = html.fromstring(page.text) # team_rows = tree.cssselect('table#confs_standings_E tbody tr th a, table#confs_standings_W tbody tr th a') team_section = tree.cssselect('table.basketball tbody') all_depth_pos = {} for section in team_section: rows = section.cssselect('tr') for depth, row in enumerate(rows): players = row.cssselect("td.depth-chart-cell a") for player in players: player_name = " ".join( player.get("href").split("/")[2].split("-")).strip() player_obj = next((player for player in current_players if player["player_name"] == player_name), None) player_depth = depth + 1 if player_obj is None: player_id = int( input("What is the player_id for " + player_name + "? ")) if player_id == 0: continue else: player_id = player_obj["player_id"] if player_id in all_depth_pos: if all_depth_pos[player_id] < player_depth: all_depth_pos[player_id] = player_depth else: all_depth_pos[player_id] = player_depth depth_rows = [] filename = './../local-data/usual_depth_pos_2017.csv' for player, depth in all_depth_pos.items(): depth_rows.append([player, depth]) csvOps.writeToCsv(depth_rows, filename)
def getSalariesForTodayAndPostToApi(): session = requests.Session() rawSalaryHtml = rw.getRawHtmlForSalaries(session) players = api.getCurrentPlayerData() salaryData = rw.extractSalaries(rawSalaryHtml, players) salaries = salaryData["currentPlayerSalaries"] newPlayerIds = salaryData["newPlayerIds"] # print(salaries) salaryResponse = api.postSalaries(salaries) newIdsResponse = api.postNewIds(newPlayerIds) # log task logger.logSalaryScrapeTaskSuccess(salaries) return len(salaries)
def getPositionChangesForDatesAndWriteToCsv(dateArr): driver = webdriver.PhantomJS() driver.set_window_size(1124, 850) playerData = api.getCurrentPlayerData() site = "fanduel" # fanduel or draftkings? allData = {'allPositionUpdates': []} # keeps track of dupes allPosChanges = [] # for each date in range: for date in dateArr: # get salary & pos data salariesForDate = fc.getSalaryDataForDate(date, site, playerData, driver) # for all new positions for date: for newPos in salariesForDate['playerPositionUpdates']: # if new pos change has already been added: if newPos in allPosChanges: continue # if not, add to changes & record else: # keep track of pos changes already logged allPosChanges.append(newPos) # create row for csv storing fullChangeRow = [ newPos["playerId"], date, newPos["newPosition"] ] allData["allPositionUpdates"].append(fullChangeRow) # posUpdates = sorted(allData['allPositionUpdates'], key=lambda k: k['playerId']) # print(allData["allPositionUpdates"]) # print(posUpdates) # posUpdatesResponse = api.updatePlayerPositions(posUpdates) # csvOps.writeToCsv(allUpdates["allPositionUpdates"], fileLoc return allData
def updateSourceIdsForPlayersManualAndLog(): ''' Requires manual input from user to match player name/ brefId ''' # get new source ids newSourceIdsByPlayer = api.getNewSourceIds() # get list of current players currentPlayers = api.getCurrentPlayerData() allPlayerNames = list((player["player_name"] for player in currentPlayers)) # arr to hold updates existingPlayerSourceIdUpdates = {} for idsByPlayer in newSourceIdsByPlayer: playerName = idsByPlayer["player_name"] # check to see if player matches exact name currentPlayerMatch = next((player for player in currentPlayers if player["player_name"] == playerName), None) # if name does not EXACTLY match any existing player: if currentPlayerMatch is None: # check similar name matches closestNameMatches = get_close_matches(playerName, allPlayerNames) # if match, prompt user to determine which one: if len(closestNameMatches) > 0: indexMatch = int( input( "Does " + playerName + " match any of these players? (Enter the number, or '0' for none.) " + str(closestNameMatches) + " ")) # if no match by name of exisiting, continue/skip, don't need to add: if indexMatch == 0: continue # finalNewPlayerObj = getFinalNewPlayerObject(playerName, idsByPlayer, sessionObj) # newPlayers.append(finalNewPlayerObj) # if match, assign to current player & add to arr elif indexMatch > 0 and indexMatch <= len(closestNameMatches): playerNameMatch = closestNameMatches[indexMatch - 1] currentPlayerMatch = next( (player for player in currentPlayers if player["player_name"] == playerNameMatch), None) newSourceIds = mapNewSourceIdsToExistingPlayer( idsByPlayer, currentPlayerMatch) existingPlayerSourceIdUpdates[ currentPlayerMatch["player_id"]] = newSourceIds # if no name matches, skip, no need to add (will be added if in lineup from RW) elif len(closestNameMatches) == 0: continue # finalNewPlayerObj = getFinalNewPlayerObject(playerName, idsByPlayer, sessionObj) # newPlayers.append(finalNewPlayerObj) # if there is an exact match: else: # skip over auto match ones, save for later continue # make source id updates (will also delete the ones that were just updated) api.updatePlayerSourceIds(existingPlayerSourceIdUpdates) logger.logSourceIdsUpdate(existingPlayerSourceIdUpdates, 'MANUAL') return existingPlayerSourceIdUpdates