def desaHashtags(fol):
"""
Funció que desa tots els hashtags en un fitxer específic pels hashtags
:param fol: El nom del directori on s'ha de guardar els hashtags
"""
hashtags = []
for h in i.getHashtags().values():
hashtags.append(h.id)
ReadWriteFiles.writeToFile(fol, ReadWriteFiles.NomFitHashtags, hashtags)
def desaUsuaris(fol):
"""
Funció que desa tots els usuaris en un fitxer específic pels usuaris
:param fol: El nom del directori on s'ha de guardar els usuaris
"""
usuaris = []
for user in i.getUsuaris().values():
usuaris.append(user.desa())
ReadWriteFiles.writeToFile(fol, ReadWriteFiles.NomFitUsuaris, usuaris)
def desaPosts(fol):
"""
Funció que desa tots els posts en un fitxer específic pels posts
:param fol: El nom del directori on s'ha de guardar els posts
"""
posts = []
for post in i.getPosts().values():
posts.append(post.desa())
ReadWriteFiles.writeToFile(fol, ReadWriteFiles.NomFitPosts, posts)
def getVhdl():
"""
S'encarrega de crear un directori amb els fitxers necessàris a dins. Si ja hi és fa les conversions Partitura-ROM-VHDL
"""
fol= ReadWriteFiles.createUseFolder()
if(not fol[1]):
return
fol = fol[0]
rom = PartituraToROM.linesToROM(fol)
ReadWriteFiles.writeToFile(fol, ReadWriteFiles.ROM, rom)
vhdl = ROMtoVHDL.getCodeFromFolder(fol)
ReadWriteFiles.writeToFile(fol, ReadWriteFiles.VHDL, vhdl)
def recuperaPosts(fol, hashtags):
"""
Funció que s'utilitza per recuperar els posts a partir del directori / fitxer on hi són guardats.
:param fol: El nom del directori on hi haurà el fitxer de Posts per poder-ne extreure la informació
:param hashtags: Diccionari de :class:`Hashtags.Hashtag` existents en la xarxa social
:return: False si no s'ha pogut fer correctament i llista de posts si sí que s'ha pogut realitzar la recuperació correctament
"""
posts = []
try:
for post in ReadWriteFiles.readlines(fol, ReadWriteFiles.NomFitPosts):
p = Post()
if (not p.recupera(post, hashtags)):
raise Exceptions.PostError()
else:
posts.append(p)
except Exceptions.PostError:
print "El post no s'ha pogut convertir correctament"
return False
except Exception as e:
print e
return False
return posts
def recupera():
"""
Funció que es crida per recuperar totes les dades a partir d'un directori. Si hi ha cap problema recuperant les dades
es demanarà que es crei una nova :class:`iTICApp.iTICApp` o es trii un altre fitxer.
"""
if (not MainLib.askYorNQuestion(
"Vol recuperar les dades d'una altra sessió? ")):
global i
i = iTICApp()
return
try:
fol = ReadWriteFiles.askFolder()
hashtags = dict((value.id, value) for value in recuperaHashtags(fol))
posts = dict(
(value.id, value) for value in recuperaPosts(fol, hashtags))
usuaris = recuperaUsuaris(fol, posts)
if (usuaris == False or posts == False or hashtags == False):
recuperaFallida()
return
except:
recuperaFallida()
return
print "Dades importades correctament"
i = iTICApp(usuaris=usuaris, posts=posts, hashtags=hashtags)
def desa():
"""
Funció que s'encarrega de guardar totes les dades de la iTICApp en el directori i fitxers corresponents
"""
fol = ReadWriteFiles.createUseFolder()
desaPosts(fol)
desaHashtags(fol)
desaUsuaris(fol)
def readFile(folder):
"""
Llegeix el fitxer dins del directori
:param folder: Directori on hi ha el fitxer ROM
:return: El txt amb tot el text
"""
l = ReadWriteFiles.readlines(folder, ReadWriteFiles.ORIGINAL)
return [line.split() for line in l] if l != False else exit()
def readFile(folder):
"""
Llegeix el fitxer dins del directori
:param folder: Directori on hi ha el fitxer ROM
:return: El txt amb tot el text
"""
l = ReadWriteFiles.readlines(folder, ReadWriteFiles.ROM)
return l if l != False else exit()
def recuperaHashtags(fol):
"""
Funció per recuperar els hashtags a partir del directori on hi ha el fitxer que guarda la informació d'aquests
:param fol: El nom del directori on hi haurà el fitxer de Posts per poder-ne extreure la informació
:return: False si no s'ha pogut fer la conversió correctament i una llista de Hashtags si s'ha pogut fer correctament
"""
hash = []
try:
for h in ReadWriteFiles.readlines(fol, ReadWriteFiles.NomFitHashtags):
hash.append(Hashtag(h))
except Exception as e:
print "Error, no s'ha pogut recuperar els hashtags correctament: ", e.message
return False
return hash
def playerid_to_playerName(playerid):
playerIDDict = ReadWriteFiles.readPlayerIDMap()
if(playerid in playerIDDict):
return playerIDDict[playerid]
else:
#print(str(playerid))
playerPage = requests.get("http://espn.go.com/nba/player/_/id/" + str(playerid))
playerTree = html.fromstring(playerPage.content)
#print(playerid)
try:
[playerName] = playerTree.xpath("//div[@class='mod-content']/*/h1/text() | //div[@class='mod-content']/h1/text()")
except (IndexError):
print("Player " + playerid + " causes error")
playerIDDict[playerid] = playerName
playerIDDict[playerName] = playerid
ReadWriteFiles.writePlayerIDDict(playerIDDict)
return playerName
def convertRotowireList(rotoWireList,urlList):
playerIDDict = ReadWriteFiles.readPlayerIDMap()
#print("\noriginal list")
#print(rotoWireList)
returnedList = []
for i in range(0,len(rotoWireList)):
rotoname = rotoWireList[i]
#takes care of cases where name matches
#and when first and last name both match -> regardless of Jr. suffix, middle name
if(rotoname in playerIDDict.keys() or (HumanName(rotoname).first + " " + HumanName(rotoname).last) in playerIDDict.keys()):
returnedList.append(rotoname)
else:
if(rotoname[1] == "."):
tempLst = []
for playerName in playerIDDict.keys():
if(HumanName(playerName).first == HumanName(rotoname).first and playerName[0] == rotoname[0]):
tempLst.append(playerName)
if len(tempLst) == 1:
returnedList.append(tempLst[0])
else:
(html.fromstring(requests.get(urlList[i]).content)).xpath("//h1[position()=1]/text()")[0]
else:
for playerName in playerIDDict.keys():
if(HumanName(playerName).first == HumanName(rotoname).first and HumanName(playerName).last == HumanName(rotoname).last):
returnedList.append(playerName)
# convertedList = [x if (x in playerIDDict.keys()) else [k for (k,v) in playerIDDict.items() if (x[1] == "." and x.split(" ")[1] in k and x[0] == k[0]) or x in k] for x in rotoWireList]
# print(returnedList)
# convertedList = [x[0] if not type(x[0]) == list else x[0][0] if len(x[0])==0 else (html.fromstring(requests.get(x[1]).content)).xpath("//h1[position()=1]/text()")[0] for x in zip(convertedList,urlList)]
#[(html.fromstring(requests.get(x).content)).xpath("//h1[position()=1]/text()")[0] for x in starterFullURLList]
#print("\nconverted list")
#print(convertedList)
return returnedList
def recuperaUsuaris(fol, posts):
"""
Funció que s'utilitza per recuperar els usuaris a partir del directori / fitxer on hi són guardats.
:param fol: El nom del directori on hi haurà el fitxer de Posts per poder-ne extreure la informació
:param posts: Diccionari de :class:`Posts.Post` existents en la xarxa social
:return: False si no s'ha pogut fer correctament i diccionari d'usuaris si sí que s'ha pogut realitzar la recuperació correctament
"""
usuaris = {}
try:
for user in ReadWriteFiles.readlines(fol,
ReadWriteFiles.NomFitUsuaris):
u = User()
if (not u.recupera(user, posts)):
raise Exceptions.UserError()
else:
usuaris[u.nick] = u
except Exceptions.UserError:
print "L'usuari no s'ha pogut convertir correctament"
return False
except Exception as e:
print e
return False
try:
for usuari in usuaris.values():
followersInstance = []
for follower in usuari.followers:
if (follower == ""):
continue
if (follower not in usuaris):
raise Exceptions.NoUserException(follower)
else:
followersInstance.append(usuaris[follower])
usuari.followers = followersInstance
except Exceptions.NoUserException as e:
print "No hi ha cap usuari anomenat", e.message, "en la xarxa social"
return False
except Exception as e:
print "Excepció al posar followers: ", e.message
return False
try:
for usuari in usuaris.values():
followingInstance = []
for following in usuari.following:
if (following == ""):
continue
if (following not in usuaris):
raise Exceptions.NoUserException(following)
else:
followingInstance.append(usuaris[following])
usuari.following = followingInstance
except Exceptions.NoUserException as e:
print "No hi ha cap usuari anomenat", e.message, "en la xarxa social"
return False
except Exception as e:
print "Excepció al posar following: ", e.message
return False
return usuaris
def createPlayerMap(gameids,currentMap):
#use defaultdict to map playerids to game stats
playerMap = defaultdict(OrderedDict)
#playerMap = OrderedDict()
print(sorted(gameids,key=lambda x: x[1]))
#will keep track of playerids of injured players on both teams
injuredIDMap = ReadWriteFiles.readInjuredIDMap()
for gameid in sorted(gameids,key=lambda x: x[1]):
gameid = gameid[0]
#print(gameid)
gameBoxScoreURL = "http://espn.go.com/nba/boxscore?gameId=" + gameid
boxScorePage = requests.get(gameBoxScoreURL)
boxScoreTree = html.fromstring(boxScorePage.content)
gameInfoURL = "http://espn.go.com/nba/game?gameId=" + gameid
gameInfoPage = requests.get(gameInfoURL)
gameInfoTree = html.fromstring(gameInfoPage.content)
#first get game data not specific to each player (time,date,score,team numbers etc)
gameDataList = []
#print(gameid)
try:
#game_time_info = boxScoreTree.xpath("//div[@class='game-time-location']/p/text()")[0] OLD ESPN
game_time_info = gameInfoTree.xpath("//div[@class='game-date-time']/span/@data-date")[0]
OTfinalStatus = gameInfoTree.xpath("//div[@class='game-status']/span/text()")[0]
if("OT" in OTfinalStatus):
[_,ot] = OTfinalStatus.split("/")
if (ot == "OT"):
overtime = 1
else:
overtime = int(ot[0])
else:
overtime = 0
#print(OTfinalStatus)
#print(game_time_info)
#[awayName,awayScore] = boxScoreTree.xpath("//div[@class='team away']/div/h3/*/text()") OLD ESPN
#[homeName,homeScore] = boxScoreTree.xpath("//div[@class='team home']/div/h3/*/text()") OLD ESPN
awayName = boxScoreTree.xpath("//div[@class='team away']/div[@class='content']/div[@class='team-container']/div[@class='team-info']/a/span[@class='short-name']/text()")[0]
homeName = boxScoreTree.xpath("//div[@class='team home']/div[@class='content']/div[@class='team-container']/div[@class='team-info']/a/span[@class='short-name']/text()")[0]
awayScore = boxScoreTree.xpath("//div[@class='team away']/div[@class='content']/div[@class='score-container']/div/text()")[0]
homeScore = boxScoreTree.xpath("//div[@class='team home']/div[@class='content']/div[@class='score-container']/div/text()")[0]
#scoreDifference = int(awayScore) - int(homeScore)
#print(scoreDifference)
#print(awayName)
#print(homeName)
#keep track of which players were on the away team and which were on the home team
#[awayTeam] = boxScoreTree.xpath("//table/thead[position()=1]/tr[@class='team-color-strip']/th/text()")
awayTeamNum = Util.team_dict[awayName]
#awayPlayeridList = boxScoreTree.xpath("//table/tbody[position()=1 or position()=2]/tr[contains(@class,'player-46')]/@class") OLD ESPN
#awayPlayeridList = [x.split("player-46-")[1] for x in awayPlayeridList]
#print(boxScoreTree.xpath("//tr/td/a/@href"))
#print(boxScoreTree.xpath("//div[@class='col column-one gamepackage-away-wrap']/*"))
#print(boxScoreTree.xpath("//div[@class='col column-one gamepackage-away-wrap']/div[@class='sub-module']/*"))
#print(boxScoreTree.xpath("//div[@class='col column-one gamepackage-away-wrap']/div[@class='sub-module']/div/table/tbody/tr/td/a/@href"))
awayPlayerURLList = boxScoreTree.xpath("//div[@class='col column-one gamepackage-away-wrap']/div[@class='sub-module']/div/table/tbody/tr/td/a/@href")
awayPlayeridList = [x.split("_/id/")[1] for x in awayPlayerURLList]
awayStarteridList = awayPlayeridList[0:5]
awayBenchidList = awayPlayeridList[5:]
#print(awayPlayeridList)
# [homeTeam] = boxScoreTree.xpath("//table/thead[position()=4]/tr[@class='team-color-strip']/th/text()")
homeTeamNum = Util.team_dict[homeName]
#homePlayeridList = boxScoreTree.xpath("//table/tbody[position()=4 or position()=5]/tr[contains(@class,'player-46')]/@class")
#homePlayeridList = [x.split("player-46-")[1] for x in homePlayeridList]
homePlayerURLList = boxScoreTree.xpath("//div[@class='col column-two gamepackage-home-wrap']/div[@class='sub-module']/div/table/tbody/tr/td/a/@href")
homePlayeridList = [x.split("_/id/")[1] for x in homePlayerURLList]
homeStarteridList = homePlayeridList[0:5]
homeBenchidList = homePlayeridList[5:]
#print(homePlayeridList)
awayInjuredIDList = []
#gets player stats for away players and appends that to the game stats
for playerid in awayStarteridList:
xPathStatsString = "//tr[td/a/@href='http://espn.go.com/nba/player/_/id/" + playerid + "']/*/text()"
xPathPositionString = "//tr[td/a/@href='http://espn.go.com/nba/player/_/id/" + playerid + "']/td/*/text()"
#print(xPathPositionString)
gameStatsList = []
# stores own team's number and also the opposing team's number
# 0 for away team then 1 for being a starter and then away score (own score) then home score (other score)
gameStatsList += Util.data_date_convert(game_time_info) + [awayTeamNum, homeTeamNum] + [0, 1, int(awayScore),int(homeScore), overtime]
playerStatsList = [boxScoreTree.xpath(xPathPositionString)[1]] + boxScoreTree.xpath(xPathStatsString)
#print(playerStatsList)
if("DNP" not in playerStatsList[1] or "COACH'S DECISION" in playerStatsList[1] or len(playerStatsList) != 2):
playerStatsList = Util.playerStatsConvert(playerStatsList)
# print(playerStatsList)
# playerMap[playerid].append(gameid)
# print(gameStatsList+playerStatsList)
playerMap[playerid][gameid]=(gameStatsList+playerStatsList)
#playerMap[playerid] = OrderedDict({gameid:(gameStatsList+playerStatsList)})
else:
awayInjuredIDList.append(playerid)
print(playerid + " is injured, so stats from game will not count")
#gets player stats for away players and appends that to the game stats
for playerid in awayBenchidList:
xPathStatsString = "//tr[td/a/@href='http://espn.go.com/nba/player/_/id/" + playerid + "']/*/text()"
xPathPositionString = "//tr[td/a/@href='http://espn.go.com/nba/player/_/id/" + playerid + "']/td/*/text()"
#print(xPathPositionString)
gameStatsList = []
# stores own team's number and also the opposing team's number
# 0 for away team then 0 for coming off the benchand then away score (own score) then home score (other score)
gameStatsList += Util.data_date_convert(game_time_info) + [awayTeamNum, homeTeamNum] + [0, 0,int(awayScore),int(homeScore), overtime]
playerStatsList = [boxScoreTree.xpath(xPathPositionString)[1]] + boxScoreTree.xpath(xPathStatsString)
#print(playerStatsList)
if("DNP" not in playerStatsList[1] or "COACH'S DECISION" in playerStatsList[1]):
playerStatsList = Util.playerStatsConvert(playerStatsList)
# print(playerStatsList)
# playerMap[playerid].append(gameid)
# print(gameStatsList+playerStatsList)
playerMap[playerid][gameid]=(gameStatsList+playerStatsList)
#playerMap[playerid] = OrderedDict({gameid:(gameStatsList+playerStatsList)})
else:
awayInjuredIDList.append(playerid)
print(playerid + " is injured, so stats from game will not count")
homeInjuredIDList = []
#gets player stats for home players and appends that to the game stats
for playerid in homeStarteridList:
xPathStatsString = "//tr[td/a/@href='http://espn.go.com/nba/player/_/id/" + playerid + "']/*/text()"
xPathPositionString = "//tr[td/a/@href='http://espn.go.com/nba/player/_/id/" + playerid + "']/td/*/text()"
#print(xPathPositionString)
gameStatsList = []
# stores own team's number and also the opposing team's number
# 1 for home team, 1 for starter and score difference is calc away score - home score
#
gameStatsList += Util.data_date_convert(game_time_info) + [homeTeamNum, awayTeamNum] + [1, 1, int(homeScore), int(awayScore), overtime]
playerStatsList = [boxScoreTree.xpath(xPathPositionString)[1]] + boxScoreTree.xpath(xPathStatsString)
if("DNP" not in playerStatsList[1] or "COACH'S DECISION" in playerStatsList[1]):
playerStatsList = Util.playerStatsConvert(playerStatsList)
# print(playerStatsList)
# playerMap[playerid].append(gameid)
playerMap[playerid][gameid]=(gameStatsList+playerStatsList)
#playerMap[playerid] = OrderedDict({gameid:(gameStatsList+playerStatsList)})
else:
homeInjuredIDList.append(playerid)
print(playerid + " is injured, so stats from game will not count")
#gets player stats for home players and appends that to the game stats
for playerid in homeBenchidList:
xPathStatsString = "//tr[td/a/@href='http://espn.go.com/nba/player/_/id/" + playerid + "']/*/text()"
xPathPositionString = "//tr[td/a/@href='http://espn.go.com/nba/player/_/id/" + playerid + "']/td/*/text()"
#print(xPathPositionString)
gameStatsList = []
# stores own team's number and also the opposing team's number
# 1 for home team, 0 for bench and score difference is calc away score - home score
#
gameStatsList += Util.data_date_convert(game_time_info) + [homeTeamNum, awayTeamNum] + [1, 0, int(homeScore), int(awayScore), overtime]
playerStatsList = [boxScoreTree.xpath(xPathPositionString)[1]] + boxScoreTree.xpath(xPathStatsString)
if("DNP" not in playerStatsList[1] or "COACH'S DECISION" in playerStatsList[1]):
playerStatsList = Util.playerStatsConvert(playerStatsList)
# print(playerStatsList)
# playerMap[playerid].append(gameid)
playerMap[playerid][gameid]=(gameStatsList+playerStatsList)
#playerMap[playerid] = OrderedDict({gameid:(gameStatsList+playerStatsList)})
else:
homeInjuredIDList.append(playerid)
print(playerid + " is injured, so stats from game will not count")
injuredIDMap[gameid] = (awayInjuredIDList,homeInjuredIDList)
except (IndexError):
print("Game " + gameid + " does not exist")
#raise IndexError
print(playerMap)
#need to UNION currentMap(defaultdict in file) and playerMap(recent games)
for playerid,orderedDict in playerMap.items():
for gameid,statList in orderedDict.items():
#TODO: look to make this more efficient
#if(not gameid in currentMap[playerid] or not playerid in currentMap):
if(not playerid in currentMap):
currentMap[playerid] = OrderedDict()
#print(type(gameid))
currentMap[playerid][gameid] = statList
return (currentMap,injuredIDMap)
def playername_to_id(playername):
playerIDDict = ReadWriteFiles.readPlayerIDMap()
return playerIDDict[playername]
import datetime
import os
import json
import numpy as np
import csv
import Optimize
import ML
import Util
import ReadWriteFiles
import Scraper
# print("Reading previously stored player-stats map")
(lastModifiedDate,currentMap) = ReadWriteFiles.readPlayerStatsFile()
isUpdated = (lastModifiedDate == datetime.date.today())
print("Getting data about players playing today")
today_playerMap = Scraper.create_todays_playerMap()
projStarters = Scraper.getProjStarters()
today_playerMap = Util.addStarting(today_playerMap,projStarters)
print(json.dumps(today_playerMap))
(lastModifiedDate,currentMap) = ReadWriteFiles.readPlayerStatsFile()
injuredTodayMap = Scraper.getInjuredPlayers()
injuredIDMap = ReadWriteFiles.readInjuredIDMap()
