def getSalariesForDatesAndPostToApi(dateArr, site):
'''
:param dateArr: array of YYYY-MM-DD date strings
:param site: 'fanduel' or 'draftkings'
:return: all new salaries & new ids in dict
'''
driver = webdriver.PhantomJS()
driver.set_window_size(1124, 850)
playerData = api.getCurrentPlayerData()
allData = {'allSalaries': [], 'allNewIds': []}
# for each date in range:
for date in dateArr:
# get salary & pos data
salariesForDate = fc.getSalaryDataForDate(date, site, playerData,
driver)
# add salary & missing player data
allData['allSalaries'].extend(salariesForDate['currentPlayerSalaries'])
allData['allNewIds'].extend(salariesForDate['missingPlayerIds'])
newSalariesResponse = api.postSalaries(allData['allSalaries'])
# UNCOMMENT LINE BELOW IF ALSO WANT TO ADD NEW FC IDS
# dedupedNewIds = [i for n, i in enumerate(allData["allNewIds"]) if i not in allData["allNewIds"][n + 1:]]
# newIdsResponse = api.postNewIds(dedupedNewIds)
return allData
def getSourceIdUpdatesForPlayersAuto():
'''
Will auto update all players w/ source ids that have an exact name match
NOTE: will override all existing source ids for any player that matches
'''
# get new source ids
newSourceIdsByPlayer = api.getNewSourceIds()
# get list of current players
currentPlayers = api.getCurrentPlayerData()
# arr to hold update
sourceIdUpdates = {}
for idsByPlayer in newSourceIdsByPlayer:
currentPlayerMatch = next(
(player for player in currentPlayers
if player["player_name"] == idsByPlayer["player_name"]), None)
# if name does not EXACTLY match any existing player:
if currentPlayerMatch is None:
continue
# if there is a match:
else:
sourceIds = mapNewSourceIdsToExistingPlayer(
idsByPlayer, currentPlayerMatch)
# add ids to player dict
sourceIdUpdates[currentPlayerMatch["player_id"]] = sourceIds
return sourceIdUpdates
def getPostGameDataForGameArr(game_arr):
all_post_game_data = []
player_data = api.getCurrentPlayerData()
session = requests.Session()
for game in game_arr:
post_game_data = post.getBoxScoreData(game["bref_slug"], session,
player_data)
post_game_data["gameId"] = game["game_id"]
all_post_game_data.append(post_game_data)
return all_post_game_data
def getDepthChartData(sessionObj):
'''
Calls rw depth scraper fcns
Parses data into dict based on new/current players
Returns dict
'''
# import team ids as dict
teamDict = jsonData.TEAM_ABBREV_TO_ID
# get
currPlayerData = api.getCurrentPlayerData()
# scrape current depth chart data & starting lineup data from rw
htmlTree = rw.getRawDepthChartTree(sessionObj, config)
currentDepths = rw.getPlayerDataFromTree(htmlTree, teamDict)
playerIdsOnRosters = []
playerDataUpdates = {"rwIdInDbUpdates": [], "rwIdNotInDbPosts": []}
# for each player in current depths, check if they are in DB:
for rwId, rwData in currentDepths.items():
try:
matchedPlayer = next(
(player
for player in currPlayerData if player["rw_id"] == rwId),
None)
rwData["playerId"] = matchedPlayer["player_id"]
# if player is already incomplete in DB, don't override w new status
if matchedPlayer["status"] == 'INCOMPLETE':
rwData["status"] = 'INCOMPLETE'
# add player id to arr for later check
playerIdsOnRosters.append(matchedPlayer["player_id"])
# only update data if player data has changed to save api calls/ db writes
if hasPlayerDataChanged(matchedPlayer, rwData):
playerDataUpdates["rwIdInDbUpdates"].append(rwData)
except: # if no match for rw_id, add player to rwNotInDbArr
rwData["rw_id"] = rwId
playerDataUpdates["rwIdNotInDbPosts"].append(rwData)
# create arr of playerIds not on a roster
playerDataUpdates["playersNotOnRoster"] = [
player["player_id"] for player in currPlayerData
if player["player_id"] not in playerIdsOnRosters
and player["status"] != 'NOT_ON_ROSTER'
]
return playerDataUpdates
def getPostGameDataForYesterdaysGames():
games = getYesterdaysGames()
playerData = api.getCurrentPlayerData()
session = requests.Session()
allPostGameData = []
for game in games:
print(game["bref_slug"])
postGameData = post.getBoxScoreData(game["bref_slug"], session,
playerData)
allPostGameData.append(postGameData)
return allPostGameData
def parseProjsFromHtml(htmlDict):
# get list of curr players from API
playerList = api.getCurrentPlayerData()
gamesToday = api.getTodaysGames()
parsedProjs = []
newIds = []
try:
# get all projection data from all raw html
nfData = nf.extractProjections(htmlDict["nf"], playerList, gamesToday)
rwData = rw.extractProjections(htmlDict["rw"], playerList, gamesToday)
fpData = fp.extractProjections(htmlDict["fp"], playerList, gamesToday)
bmData = bm.extractProjections(htmlDict["bm"], playerList, gamesToday)
# parsedProjs = bmData["projections"]
# newIds = bmData["newPlayerIds"]
# concat all projs & missing players into single arrays
parsedProjs = nfData["projections"] + rwData["projections"] + fpData["projections"] + bmData["projections"]
newIds = nfData["newPlayerIds"] + rwData["newPlayerIds"] + fpData["newPlayerIds"] + bmData["newPlayerIds"]
counts = {
"nf": {
"projs": len(nfData["projections"]),
"new_ids": len(nfData["newPlayerIds"]),
"total_rows": nfData["totalNumRows"]
},
"rw": {
"projs": len(rwData["projections"]),
"new_ids": len(rwData["newPlayerIds"]),
"total_rows": rwData["totalNumRows"]
},
"fp": {
"projs": len(fpData["projections"]),
"new_ids": len(fpData["newPlayerIds"]),
"total_rows": fpData["totalNumRows"]
},
"bm": {
"projs": len(bmData["projections"]),
"new_ids": len(bmData["newPlayerIds"]),
"total_rows": bmData["totalNumRows"]
},
}
except Exception as error:
# TODO: handle this err. Log? Throw?
print("COULDN'T PARSE PROJECTIONS, ERROR:", error)
return {'projections': parsedProjs, 'newPlayerIds': newIds, 'counts': counts}
def scrapeGameStats(gameOrMinDate, maxDate=None):
if maxDate == None:
gamesToScrape = api.getGamesForDate(gameOrMinDate)
else:
gamesToScrape = api.getGamesInRange(gameOrMinDate, maxDate)
playerList = api.getCurrentPlayerData()
session = requests.Session()
allGameStats = []
for game in gamesToScrape:
statsForGame = actual.getGameStats(game, playerList, session)
allGameStats.extend(statsForGame)
return allGameStats
def get_usual_depth_positions(season):
session = requests.Session()
current_players = api.getCurrentPlayerData()
# teams_url = 'http://www.basketball-reference.com/leagues/NBA_' + str(season) + '.html'
depth_url = 'http://basketball.realgm.com/nba/depth-charts/' + str(season)
page = session.get(depth_url)
tree = html.fromstring(page.text)
# team_rows = tree.cssselect('table#confs_standings_E tbody tr th a, table#confs_standings_W tbody tr th a')
team_section = tree.cssselect('table.basketball tbody')
all_depth_pos = {}
for section in team_section:
rows = section.cssselect('tr')
for depth, row in enumerate(rows):
players = row.cssselect("td.depth-chart-cell a")
for player in players:
player_name = " ".join(
player.get("href").split("/")[2].split("-")).strip()
player_obj = next((player for player in current_players
if player["player_name"] == player_name),
None)
player_depth = depth + 1
if player_obj is None:
player_id = int(
input("What is the player_id for " + player_name +
"? "))
if player_id == 0:
continue
else:
player_id = player_obj["player_id"]
if player_id in all_depth_pos:
if all_depth_pos[player_id] < player_depth:
all_depth_pos[player_id] = player_depth
else:
all_depth_pos[player_id] = player_depth
depth_rows = []
filename = './../local-data/usual_depth_pos_2017.csv'
for player, depth in all_depth_pos.items():
depth_rows.append([player, depth])
csvOps.writeToCsv(depth_rows, filename)
def getSalariesForTodayAndPostToApi():
session = requests.Session()
rawSalaryHtml = rw.getRawHtmlForSalaries(session)
players = api.getCurrentPlayerData()
salaryData = rw.extractSalaries(rawSalaryHtml, players)
salaries = salaryData["currentPlayerSalaries"]
newPlayerIds = salaryData["newPlayerIds"]
# print(salaries)
salaryResponse = api.postSalaries(salaries)
newIdsResponse = api.postNewIds(newPlayerIds)
# log task
logger.logSalaryScrapeTaskSuccess(salaries)
return len(salaries)
def getPositionChangesForDatesAndWriteToCsv(dateArr):
driver = webdriver.PhantomJS()
driver.set_window_size(1124, 850)
playerData = api.getCurrentPlayerData()
site = "fanduel" # fanduel or draftkings?
allData = {'allPositionUpdates': []}
# keeps track of dupes
allPosChanges = []
# for each date in range:
for date in dateArr:
# get salary & pos data
salariesForDate = fc.getSalaryDataForDate(date, site, playerData,
driver)
# for all new positions for date:
for newPos in salariesForDate['playerPositionUpdates']:
# if new pos change has already been added:
if newPos in allPosChanges:
continue
# if not, add to changes & record
else:
# keep track of pos changes already logged
allPosChanges.append(newPos)
# create row for csv storing
fullChangeRow = [
newPos["playerId"], date, newPos["newPosition"]
]
allData["allPositionUpdates"].append(fullChangeRow)
# posUpdates = sorted(allData['allPositionUpdates'], key=lambda k: k['playerId'])
# print(allData["allPositionUpdates"])
# print(posUpdates)
# posUpdatesResponse = api.updatePlayerPositions(posUpdates)
# csvOps.writeToCsv(allUpdates["allPositionUpdates"], fileLoc
return allData
def updateSourceIdsForPlayersManualAndLog():
'''
Requires manual input from user to match player name/ brefId
'''
# get new source ids
newSourceIdsByPlayer = api.getNewSourceIds()
# get list of current players
currentPlayers = api.getCurrentPlayerData()
allPlayerNames = list((player["player_name"] for player in currentPlayers))
# arr to hold updates
existingPlayerSourceIdUpdates = {}
for idsByPlayer in newSourceIdsByPlayer:
playerName = idsByPlayer["player_name"]
# check to see if player matches exact name
currentPlayerMatch = next((player for player in currentPlayers
if player["player_name"] == playerName),
None)
# if name does not EXACTLY match any existing player:
if currentPlayerMatch is None:
# check similar name matches
closestNameMatches = get_close_matches(playerName, allPlayerNames)
# if match, prompt user to determine which one:
if len(closestNameMatches) > 0:
indexMatch = int(
input(
"Does " + playerName +
" match any of these players? (Enter the number, or '0' for none.) "
+ str(closestNameMatches) + " "))
# if no match by name of exisiting, continue/skip, don't need to add:
if indexMatch == 0:
continue
# finalNewPlayerObj = getFinalNewPlayerObject(playerName, idsByPlayer, sessionObj)
# newPlayers.append(finalNewPlayerObj)
# if match, assign to current player & add to arr
elif indexMatch > 0 and indexMatch <= len(closestNameMatches):
playerNameMatch = closestNameMatches[indexMatch - 1]
currentPlayerMatch = next(
(player for player in currentPlayers
if player["player_name"] == playerNameMatch), None)
newSourceIds = mapNewSourceIdsToExistingPlayer(
idsByPlayer, currentPlayerMatch)
existingPlayerSourceIdUpdates[
currentPlayerMatch["player_id"]] = newSourceIds
# if no name matches, skip, no need to add (will be added if in lineup from RW)
elif len(closestNameMatches) == 0:
continue
# finalNewPlayerObj = getFinalNewPlayerObject(playerName, idsByPlayer, sessionObj)
# newPlayers.append(finalNewPlayerObj)
# if there is an exact match:
else:
# skip over auto match ones, save for later
continue
# make source id updates (will also delete the ones that were just updated)
api.updatePlayerSourceIds(existingPlayerSourceIdUpdates)
logger.logSourceIdsUpdate(existingPlayerSourceIdUpdates, 'MANUAL')
return existingPlayerSourceIdUpdates