def __init__(self,globalNum):
logger.info("Initialising MetaSteam");
#Locks:
#The lock for reading and writing the output json of game data
self.jsonLock = threading.Lock()
#the lock for the runtime objects of game data
self.internalDataLock = threading.Lock()
#Data:
self.userName = "******"
self.globalNumberOfGamesToSearch = int(globalNum)
#Steam Executable Location:
self.steamLocation = None
#Steam Libraries:
self.libraryLocations =[]
#Location of meta steam program:
if isFrozen():
self.programLocation = os.path.dirname(unicode(sys.executable,sys.getfilesystemencoding()))
else:
self.programLocation = os.path.dirname(unicode(__file__,sys.getfilesystemencoding()))
logger.info("programLocation="+self.programLocation)
#The location of the settings json data:
self.settingsFile = os.path.join(self.programLocation,"data","settings.json")
#Game information:
self.installedGames = {} #key = appid
self.profileGames = {} #key = appid
#initialisation:
self.loadSettingsFromJson()
self.findLibraries()
self.findSteam()
#Web Scrapers:
self.scraper = SteamStoreScraper()
self.profileScraper = SteamProfileScraper(self.userName)
#TODO: verify locations
self.verifyLocations()
#import already scraped gameData
self.importFromJson()
if not self.profileGames:
self.getProfileGames()
else:
logger.info("Skipping Profile Scrape")
self.loadGamesFromManifests()
self.exportToJson()
#By this point, all previously found games
#should be loaded. Now just find new games,
#and get info for them
logger.info("Setting up web info extraction thread")
getInfoThread = threading.Thread(target=MetaSteam.getInfoForAllGames,args=(self,))
logger.info("Starting web info extraction thread")
getInfoThread.start()
def compare_to_user(username):
logger.info( "TODO: allow comparison of user profiles")
profileScraper = SteamProfileScraper(username)
extractedInfo = profileScraper.scrape()
return extractedInfo
class MetaSteam:
'''
@class MetaSteam
@method __init__
@purpose constructor
@param globalNum The max number of games to scrape in a session
'''
def __init__(self,globalNum):
logger.info("Initialising MetaSteam");
#Locks:
#The lock for reading and writing the output json of game data
self.jsonLock = threading.Lock()
#the lock for the runtime objects of game data
self.internalDataLock = threading.Lock()
#Data:
self.userName = "******"
self.globalNumberOfGamesToSearch = int(globalNum)
#Steam Executable Location:
self.steamLocation = None
#Steam Libraries:
self.libraryLocations =[]
#Location of meta steam program:
if isFrozen():
self.programLocation = os.path.dirname(unicode(sys.executable,sys.getfilesystemencoding()))
else:
self.programLocation = os.path.dirname(unicode(__file__,sys.getfilesystemencoding()))
logger.info("programLocation="+self.programLocation)
#The location of the settings json data:
self.settingsFile = os.path.join(self.programLocation,"data","settings.json")
#Game information:
self.installedGames = {} #key = appid
self.profileGames = {} #key = appid
#initialisation:
self.loadSettingsFromJson()
self.findLibraries()
self.findSteam()
#Web Scrapers:
self.scraper = SteamStoreScraper()
self.profileScraper = SteamProfileScraper(self.userName)
#TODO: verify locations
self.verifyLocations()
#import already scraped gameData
self.importFromJson()
if not self.profileGames:
self.getProfileGames()
else:
logger.info("Skipping Profile Scrape")
self.loadGamesFromManifests()
self.exportToJson()
#By this point, all previously found games
#should be loaded. Now just find new games,
#and get info for them
logger.info("Setting up web info extraction thread")
getInfoThread = threading.Thread(target=MetaSteam.getInfoForAllGames,args=(self,))
logger.info("Starting web info extraction thread")
getInfoThread.start()
'''
@class MetaSteam
@method findLibraries
@purpose To find, across all drives, steam directories for search later
@modifies MetaSteam.libraryLocations
'''
def findLibraries(self):
if skipWin: return
try:
logger.info("finding libraries")
drives = win32api.GetLogicalDriveStrings()
logger.info("Drives:" + str(drives))
drives = drives.split('\000')[:-1]
for drive in drives:
logger.info( "Checking Drive: " + drive)
loc = os.path.join(drive,"*")
result = glob.glob(loc)
#TODO: check lower case behaviour for windows
potentials = [s for s in result if "steam" in s.lower()]
#if any("steam" in s.lower() for s in result):
for potential in potentials:
folder = os.path.join(potential,"steamapps")
if os.path.exists(folder):
logger.info( "Found: " + folder)
self.libraryLocations.append(folder)
else:
logger.warn("Steam Location Doesnt Exist: " + folder)
#raise MetaSteamException("Steam Location doesnt exist: " + folder)
except Exception as e:
logger.error("Exception: findLibraries: " + str(e))
'''
@class MetaSteam
@method findSteam
@purpose finds the steam executable
@modifies MetaSteam.steamLocation
@modifies MetaSteam.libraryLocations
'''
def findSteam(self):
if skipWin: return
try:
logger.info("Finding Steam")
#if a steam location was loaded from settings:
if self.steamLocation and os.path.exists(self.steamLocation):
self.libraryLocations.append(os.path.join(self.steamLocation.replace('Steam.exe',''),"steamapps"))
return
else:
logger.warn("Steam Location unknown: " + self.steamLocation)
logger.warn("Attempting hard coded location")
steamLocation = os.path.join("C:\\","Program Files (x86)","Steam")
logger.info("Steam Location: " + steamLocation)
if os.path.exists(steamLocation):
logger.info("Steam Location Exists")
self.steamLocation = steamLocation
self.libraryLocations.append(os.path.join(steamLocation,"steamapps"))
return
else:
logger.warn("Couldn't find Steam")
#raise MetaSteamException("Default Steam Location doesnt exist")
#final fallback:
self.steamLocation = None
except Exception as e:
self.steamLocation = None
logger.error("Exception: findSteam: " + str(e))
'''
@class MetaSteam
@method verifyLocations
@purpose verifies that steamlocation and steamLibraries exist
'''
def verifyLocations(self):
for location in self.libraryLocations:
if not os.path.exists(location):
logger.warn("Removing non-existent location: " + location)
self.libraryLocations.remove(location)
if self.steamLocation is not None and not os.path.exists(self.steamLocation):
logger.warn("steamLocation doesnt exist: " + self.steamLocation)
self.steamLocation = None
#check the steamLocation points to the exe:
if self.steamLocation is not None and ".exe" not in self.steamLocation:
logger.warn("steamLocation isnt an exe: " + self.steamLocation)
self.steamLocation = None
'''
@class MetaSteam
@method exportToJson
@purpose Exports the found game data, on drive and web scraped, to a file
@threadSafe
'''
def exportToJson(self):
try:
logger.info("Exporting to Json")
if not os.path.exists(os.path.join(self.programLocation,"data")):
raise MetaSteamException("No Data Directory Exists")
self.jsonLock.acquire()
self.internalDataLock.acquire()
outputFile = open(os.path.join(self.programLocation,"data","gameData.json"),'w')
combinedData = {}
combinedData['installed'] = self.installedGames
combinedData['profile'] = self.profileGames
outputJson = json.dump(combinedData,outputFile,sort_keys=True, indent=4, separators=(','':'),ensure_ascii=True, skipkeys=True)
except Exception as e:
logger.error("Exception: exportToJson: " + str(e))
finally:
outputFile.close()
self.internalDataLock.release()
self.jsonLock.release()
'''
@class MetaSteam
@method importFromJson
@purpose Loads a json file that has information about games
@modifies MetaSteam.installedGames
@modifies MetaSteam.profileGames
@threadSafe
'''
def importFromJson(self):
inputFile = None
try:
self.jsonLock.acquire()
self.internalDataLock.acquire()
logger.info("Loading Game Data Json")
inputFile = codecs.open(os.path.join(self.programLocation, "data","gameData.json"))
importedJson = json.load(inputFile)
#
for game in importedJson['installed'].values():
self.installedGames[game['appid']] = game
#
for game in importedJson['profile'].values():
self.profileGames[game['appid']] = game
except Exception as e:
logger.error("Exception: importFromJson: " + str(e))
finally:
if inputFile:
inputFile.close()
self.jsonLock.release()
self.internalDataLock.release()
'''
@class MetaSteam
@method loadSettingsFromJson
@purpose To load external settings for username,steamlocation,librarylocations
@modifies MetaSteam.userName
@modifies MetaSteam.steamLocation
@modifies MetaSteam.libraryLocations
'''
def loadSettingsFromJson(self):
try:
logger.info("Loading settings from json")
inputFile = codecs.open(self.settingsFile)
importedJson = json.load(inputFile)
self.userName = importedJson['steamProfileName']
logger.info("Loaded username:"******"Loaded steam location:" + self.steamLocation)
self.libraryLocations = importedJson['steamLibraryLocations']
logger.info("Loaded libraryLocations:" + str(self.libraryLocations))
#todo: deal with web browser
except Exception as e:
logger.exception("Exception: loadSettingsFromJson: " + str(e))
finally:
inputFile.close()
'''
@class MetaSteam
@method loadGamesFromManifest
@purpose find all installed manifests in libraryLocations, parse each manifest
'''
def loadGamesFromManifests(self):
try:
logger.info("Loading Games")
for folder in self.libraryLocations:
manifests = glob.glob(os.path.join(folder,"*.acf"))
logger.info(str(folder) + " : Number of found manifests: " + str(len(manifests)))
for manifest in manifests:
self.parseManifest(manifest)
except Exception as e:
logger.error("Exception: loadGamesFromManifest: " + str(e))
'''
@class MetaSteam
@method parseManifest
@purpose To take an installed game manifest and extract info from it
@modifies MetaSteam.installedGames[gameid]
@threadSafe
'''
def parseManifest(self,manifest):
try:
#logger.info("Parsing a manifest")
f = open(manifest,'r')
regex = re.compile('"(.+?)"\s+"(.+?)"')
data = {}
for line in f:
line = unicode(line,errors="ignore")
#logger.info("Line type: " + str(type(line)))
match = regex.search(line)
if match:
data[match.group(1)] = match.group(2)
gameid = data['appid']
#logger.info("Found: " + data['name'])
#logger.info("TYPE: " + str(type(gameid)))
data['__Installed'] = True
#if it doesnt exist yet:
if not gameid in self.installedGames.keys():
self.internalDataLock.acquire()
self.installedGames[gameid] = data
self.internalDataLock.release()
else:
#if it exists already copy over information
self.internalDataLock.acquire()
for field in data.keys():
self.installedGames[gameid][field] = data[field]
self.internalDataLock.release()
except Exception as e:
logger.error("Exception: parseManifest: " + str(e))
finally:
f.close()
'''
@class MetaSteam
@method combineData
@todo
@purpose To fold the two forms of stored data together
'''
def combineData(self):
print("TODO:combine data from store scraping and profile scraping")
logger.warn("Combine Data not implemented")
'''
@class MetaSteam
@method getProfileGames
@purpose Call the profile scraper, to extract all information about a users games, not just installed games
@modifies MetaSteam.profileGames
@threadSafe
'''
def getProfileGames(self):
try:
logger.info( "Getting Profile Games")
extractedInfo = self.profileScraper.scrape()
self.internalDataLock.acquire()
self.profileGames = extractedInfo
self.internalDataLock.release()
except Exception as e:
logger.error("Exception: getProfileGames: " + str(e))
'''
@class MetaSteam
@method getInfoForGame
@purpose To store scrape a specific game, and update the passed in game's data
@para game A Game dictionary, likely built from a parsed manifest
'''
#get steam page tags and release date
def getInfoForGame(self,game):
logger.info("Getting Info for Game: " + str(game['appid']))
try:
extractedInfo = self.scraper.scrape(game['appid'])
game['__tags'] = extractedInfo[0]
game['releaseDate'] = extractedInfo[1]
game['__description'] = extractedInfo[2]
game['__review'] = extractedInfo[3]
game['__developer'] = extractedInfo[4]
game['__publisher'] = extractedInfo[5]
game['__scraped'] = True
except Exception as e:
logger.warn("Exception: getInfoForGame: " + str(e))
finally:
return game
'''
@class MetaSteam
@method getInfoForAllGames
@purpose for all games installed, scrape information about them
@todo be able to reset __scraped
@threadSafe
'''
def getInfoForAllGames(self):
try:
logger.info("Getting web information for all games")
scrapedGames = []
for game in self.installedGames.values():
if self.globalNumberOfGamesToSearch < 1: break
if '__scraped' in game:
continue
try:
self.internalDataLock.acquire()
self.installedGames[game['appid']] = self.getInfoForGame(game)
except Exception as e:
logger.error("Exception: getInfoForAllGames: " + game['appid'] + str(e))
finally:
self.internalDataLock.release()
if 'name' in game.keys():
logger.info("Game: " + game['name'] + " parsed")
game['__scraped'] = True
self.exportToJson()
self.globalNumberOfGamesToSearch -= 1
scrapedGames.append(game['name'])
time.sleep(waitTime)
logger.info( "Have scanned all games for this run: " + str(len(scrapedGames)))
except Exception as e:
logger.error("Exception: getInfoForAllGames: " + str(e))
finally:
self.exportToJson()
'''
@class MetaSteam
@method loadVisualisation
@purpose Starts the http server and opens the web browser to the metaSteam page
'''
def loadVisualisation(self):
try:
#Start the web server in a separate thread
logger.info( "Sending to RunLocalServer: " + str(self))
serverThread = threading.Thread(target=MetaSteamHTTPServer.runLocalServer,args=(self,))
serverThread.start()
logger.info( "\nOPENING WEBBROWSER:\n\n")
webbrowser.open("http:\\localhost:8000\web\MetaSteam.html")
except Exception as e:
logger.error("Exception: loadVisualisation: " + str(e))
'''
@class MetaSteam
@method startGame
@purpose Calls the steam executable to start the specified game
@param appid the steam appid of the game you want to start
'''
def startGame(self,appid):
try:
if self.steamLocation is None:
logger.warn("Tried to start game, but there is no registered steam executable")
return False
logger.info( "Starting game: " + str(appid))
if appid == None:
appid = 440
logger.warn( "No Appid, defaulting to TF2")
logger.info("Opening using: " + self.steamLocation)
subprocess.call([self.steamLocation,"-applaunch",str(appid)])
return True
except Exception as e:
logger.error("Exception: startGame: " + str(e))
finally:
return False