def get_scraper_by_name(self, sname): """Given a scraper name, returns the scraper class Args: sname: Name of the scraper, e.g. thegamesdb.net or MAME Raises: ConfigScraperSiteDoesNotExistException: No scraper matches the name """ try: target = self.scrapers[sname] except KeyError as e: raise ConfigScraperSiteDoesNotExistException("Unsupported scraper: {0}".format(sname)) log.debug("Instantiating scraper class {0} - {1}".format(sname, target)) try: module = __import__(target.lower()) class_ = getattr(module, target) instance = class_() except ImportError: log.error("Unable to find scraper {0}".format(sname)) raise return instance
def onClick(self, controlId): log.debug("Begin onClick UIGameInfoView") if (controlId == CONTROL_BUTTON_PLAYGAME): self.launchEmu() log.debug("End onClick UIGameInfoView")
def retrieve(self, gameid, platform): result = {} if not xbmcvfs.exists(self.nfo_file): return result game = ET.ElementTree() if sys.version_info >= (2, 7): parser = ET.XMLParser(encoding='utf-8') else: parser = ET.XMLParser() game.parse(self.nfo_file, parser) # Standard fields for k, v in self._game_mapping.items(): # HACK - This is only used to retain backwards compatibility with existing scraper, where each key value was a # list, even if only one value is in that list try: result[k] = [game.find(v).text] # FIXME TODO When we remove the hack, this will be the code to use: # result[k] = game.find(v).text except Exception as e: # Typically this result doesn't have this field log.debug("Unable to extract data from key {0}".format(k)) # Custom fields result['Genre'] = self._parse_genres(game) return result
def retrieve(self, gameid, platform=None): result = {} fh = xbmcvfs.File(self._get_xml_path()) tree = ET.fromstring(fh.read()) fh.close() #gameid is the exact name of the game used in <description> or @name (for platforms with MAME style names) searchpattern = './/game[description="%s"]' % gameid if platform in self._MAME_style_platforms: searchpattern = './/game[@name="%s"]' % gameid game = tree.find(searchpattern) # Standard fields for k, v in self._game_mapping.items(): # HACK - This is only used to retain backwards compatibility with existing scraper, where each key value was a # list, even if only one value is in that list try: result[k] = [game.find(v).text] # FIXME TODO When we remove the hack, this will be the code to use: # result[k] = game.find(v).text except Exception: # Typically this result doesn't have this field log.debug("Unable to extract data from key {0}".format(k)) # Custom fields result['Genre'] = self._parse_genres(game) return result
def insertFile(self, fileName, gameId, fileType, romCollectionId, publisherId, developerId): log.debug("Begin Insert file: %s" % fileName) parentId = None # TODO console and romcollection could be done only once per RomCollection # fileTypeRow[3] = parent if fileType.parent == 'game': parentId = gameId elif fileType.parent == 'romcollection': parentId = romCollectionId elif fileType.parent == 'publisher': parentId = publisherId elif fileType.parent == 'developer': parentId = developerId log.info("Inserting file with parent {0} (type {1})".format(parentId, fileType.parent)) fileRow = File(self.gdb).getFileByNameAndTypeAndParent(fileName, fileType.id, parentId) if fileRow is None: log.info("File does not exist in database. Insert file: %s" % fileName) f = File(self.gdb) try: f.insert((fileName, fileType.id, parentId)) except Exception: log.warn("Error inserting into database: %s" % fileName) else: log.info("File already exists in database: %s" % fileName)
def insertFile(self, fileName, gameId, fileType, romCollectionId, publisherId, developerId): log.debug("Begin Insert file: %s" % fileName) parentId = None # TODO console and romcollection could be done only once per RomCollection # fileTypeRow[3] = parent if fileType.parent == 'game': parentId = gameId elif fileType.parent == 'romcollection': parentId = romCollectionId elif fileType.parent == 'publisher': parentId = publisherId elif fileType.parent == 'developer': parentId = developerId log.info("Inserting file with parent {0} (type {1})".format(parentId, fileType.parent)) fileRow = File(self.gdb).getFileByNameAndTypeAndParent(fileName, fileType.id, parentId) if fileRow is None: log.info("File does not exist in database. Insert file: %s" % fileName) f = File(self.gdb) try: f.insert((fileName, fileType.id, parentId)) except Exception, (exc): log.warn("Error inserting into database: %s" % fileName)
def open_json_url(self, **kwargs): log.info('Retrieving url %s, params = %s' % (kwargs['url'], kwargs['params'])) try: r = requests.get(kwargs['url'], headers=self._headers, params=kwargs['params']) except ValueError as e: # Typically non-JSON response raise ScraperUnexpectedContentException( "Non-JSON response received") log.debug(u"Retrieving {0} as JSON - HTTP{1}".format( r.url, r.status_code)) if r.status_code == 401: # Mobygames and GiantBomb send a 401 if the API key is invalid raise ScraperUnauthorisedException("Invalid API key sent") if r.status_code == 429: raise ScraperExceededAPIQuoteException( "Scraper exceeded API key limits") if r.status_code == 500: raise ScraperWebsiteUnavailableException("Website unavailable") return r.json()
def __checkGameHasSaveStates(self, gameRow, filenameRows): if self.romCollection.saveStatePath == '': log.debug("No save state path set") return '' rom = filenameRows[0][0] saveStatePath = self.__replacePlaceholdersInParams(self.romCollection.saveStatePath, rom, gameRow) saveStateFiles = glob.glob(saveStatePath) if len(saveStateFiles) == 0: log.debug("No save state files found") return '' log.info('saveStateFiles found: ' + str(saveStateFiles)) # don't select savestatefile if ASKNUM is requested in Params if re.search('(?i)%ASKNUM%', self.romCollection.saveStateParams): return saveStateFiles[0] options = [util.localize(32165)] for f in saveStateFiles: options.append(os.path.basename(f)) selectedFile = xbmcgui.Dialog().select(util.localize(32166), options) # If selections is canceled or "Don't launch statefile" option if selectedFile < 1: return '' return saveStateFiles[selectedFile - 1]
def __audioSuspend(self): if __addon__.getSetting( util.SETTING_RCB_SUSPENDAUDIO).upper() == 'TRUE': log.debug("Suspending audio") xbmc.executebuiltin("PlayerControl(Stop)") xbmc.enableNavSounds(False) xbmc.audioSuspend()
def getArtworkForGame(self, romCollection, gamename, gamenameFromFile, gamedescription, foldername, publisher, developer): artWorkFound = False artworkfiles = {} artworkurls = {} for path in romCollection.mediaPaths: log.info("FileType: {0}".format(path.fileType.name)) # TODO replace %ROMCOLLECTION%, %PUBLISHER%, ... fileName = path.path.replace("%GAME%", gamenameFromFile) continueUpdate, artworkurls = self.getThumbFromOnlineSource(gamedescription, path.fileType.name, fileName, artworkurls) if not continueUpdate: return False, {}, {} log.debug("Additional data path: %s" % path.path) files = self.resolvePath((path.path,), gamename, gamenameFromFile, foldername, romCollection.name, publisher, developer) if len(files) > 0: artWorkFound = True else: self.missingArtworkFile.add_entry(gamename, gamenameFromFile, path.fileType.name) artworkfiles[path.fileType] = files return artWorkFound, artworkfiles, artworkurls
def __get_archive_file_name(self, filename): log.info("ArchiveHandler.__get_archive_file_name") archive_file_name = 'archive://%(archive_file)s' % { 'archive_file': quote_plus(xbmcvfs.translatePath(filename)) } log.debug("archive_file_name: {0}".format(archive_file_name)) return archive_file_name
def __extract_files(self, archive_file, filenames, directory_to): log.info("ArchiveHandler.__extract_files") files_out = list() if self.__is_7z_on_windows(archive_file): return self.__getArchives7z(archive_file, filenames, directory_to) archive_path = self.__get_archive_file_name(archive_file) for ff in filenames: file_from = os.path.join(archive_path, ff).replace('\\', '/') success = xbmcvfs.copy( file_from, os.path.join(xbmcvfs.translatePath(directory_to), ff)) if not success: log.error( 'Error extracting file %(ff)s from archive %(archive_file)s' % { 'ff': ff, 'archive_file': archive_file }) else: log.debug( 'Extracted file %(ff)s from archive %(archive_file)s' % { 'ff': ff, 'archive_file': archive_file }) files_out.append( os.path.join(xbmcvfs.translatePath(directory_to), ff)) return files_out
def get_scraper_by_name(self, sname): """Given a scraper name, returns the scraper class Args: sname: Name of the scraper, e.g. thegamesdb.net or MAME Raises: ConfigScraperSiteDoesNotExistException: No scraper matches the name """ try: target = self.scrapers[sname] except KeyError as e: raise ConfigScraperSiteDoesNotExistException( "Unsupported scraper: {0}".format(sname)) log.debug("Instantiating scraper class {0} - {1}".format( sname, target)) try: module = __import__(target.lower()) class_ = getattr(module, target) instance = class_() except ImportError: log.error("Unable to find scraper {0}".format(sname)) raise return instance
def add_romfiles_to_db(self, romFiles, gameId): for romFile in romFiles: log.debug("Adding romfile to DB: %s" % romFile) fileType = FileType() fileType.id, fileType.name, fileType.parent = 0, "rcb_rom", "game" self.insertFile(romFile, gameId, fileType, None, None, None) del fileType
def retrieve(self, gameid, platform): result = {} if not xbmcvfs.exists(self.nfo_file): return result fh = xbmcvfs.File(self.nfo_file) game = ET.fromstring(fh.read()) fh.close() # Standard fields for k, v in self._game_mapping.items(): # HACK - This is only used to retain backwards compatibility with existing scraper, where each key value was a # list, even if only one value is in that list try: result[k] = [game.find(v).text] # FIXME TODO When we remove the hack, this will be the code to use: # result[k] = game.find(v).text except Exception: # Typically this result doesn't have this field log.debug("Unable to extract data from key {0}".format(k)) # Custom fields result['Genre'] = self._parse_genres(game) return result
def readImagePlacing(self, imagePlacingName, tree): #fileTypeForRow = None fileTypeForRows = tree.findall('ImagePlacing/fileTypeFor') fileTypeForRow = next((element for element in fileTypeForRows if element.attrib.get('name') == imagePlacingName), None) if fileTypeForRow is None: Logutil.log('Configuration error. ImagePlacing/fileTypeFor %s does not exist in config.xml' % str(imagePlacingName), util.LOG_LEVEL_ERROR) return None, util.localize(32005) imagePlacing = ImagePlacing() imagePlacing.name = imagePlacingName for attr in ['fileTypesForGameList', 'fileTypesForGameListSelected', 'fileTypesForMainView1', 'fileTypesForMainView2', 'fileTypesForMainView3', 'fileTypesForMainViewBackground', 'fileTypesForMainViewGameInfoBig', 'fileTypesForMainViewGameInfoUpperLeft', 'fileTypesForMainViewGameInfoUpperRight', 'fileTypesForMainViewGameInfoLowerLeft', 'fileTypesForMainViewGameInfoLowerRight', 'fileTypesForMainViewGameInfoLower', 'fileTypesForMainViewGameInfoUpper', 'fileTypesForMainViewGameInfoRight', 'fileTypesForMainViewGameInfoLeft', 'fileTypesForMainViewVideoWindowBig', 'fileTypesForMainViewVideoWindowSmall', 'fileTypesForMainViewVideoFullscreen']: # Hack - class attribute fileTypesForXXX doesn't match XML key fileTypeForXXX val = self.readFileTypeForElement(fileTypeForRow, attr.replace('fileTypesFor', 'fileTypeFor'), tree) log.debug("Reading imageplacing for {0}: {1}".format(attr, val)) setattr(imagePlacing, attr, val) return imagePlacing, ''
def __copyLauncherScriptsToUserdata(self): log.info('__copyLauncherScriptsToUserdata') oldBasePath = os.path.join(util.getAddonInstallPath(), 'resources', 'scriptfiles') newBasePath = os.path.join(util.getAddonDataPath(), 'scriptfiles') files = [] # Copy applaunch shell script/batch file if self.env == 'win32': files.append('applaunch.bat') else: files.append('applaunch.sh') # Copy VBS files if self.env == 'win32' and __addon__.getSetting( util.SETTING_RCB_USEVBINSOLOMODE).lower() == 'true': files += ['applaunch-vbs.bat', 'LaunchKodi.vbs', 'Sleep.vbs'] for f in files: if not xbmcvfs.exists(os.path.join(newBasePath, f)): log.debug("Copying file {0} from {1} to {2}".format( f, oldBasePath, newBasePath)) if not xbmcvfs.copy(os.path.join(oldBasePath, f), os.path.join(newBasePath, f)): log.warn("Error copying file")
def __checkGameHasSaveStates(self, gameRow, filenameRows): if self.romCollection.saveStatePath == '': log.debug("No save state path set") return '' rom = filenameRows[0][0] saveStatePath = self.__replacePlaceholdersInParams( self.romCollection.saveStatePath, rom, gameRow) saveStateFiles = glob.glob(saveStatePath) if len(saveStateFiles) == 0: log.debug("No save state files found") return '' log.info('saveStateFiles found: ' + str(saveStateFiles)) # don't select savestatefile if ASKNUM is requested in Params if re.search('(?i)%ASKNUM%', self.romCollection.saveStateParams): return saveStateFiles[0] options = [util.localize(32165)] for f in saveStateFiles: options.append(os.path.basename(f)) selectedFile = xbmcgui.Dialog().select(util.localize(32166), options) # If selections is canceled or "Don't launch statefile" option if selectedFile < 1: return '' return saveStateFiles[selectedFile - 1]
def retrieve(self, gameid, platform=None): result = {} tree = ET.ElementTree() if sys.version_info >= (2, 7): parser = ET.XMLParser(encoding='utf-8') else: parser = ET.XMLParser() tree.parse(self._get_xml_path(), parser) #gameid is the exact name of the game used in element <description> game = tree.find('.//game[description="%s"]'%gameid) # Standard fields for k, v in self._game_mapping.items(): # HACK - This is only used to retain backwards compatibility with existing scraper, where each key value was a # list, even if only one value is in that list try: result[k] = [game.find(v).text] # FIXME TODO When we remove the hack, this will be the code to use: # result[k] = game.find(v).text except Exception as e: # Typically this result doesn't have this field log.debug("Unable to extract data from key {0}".format(k)) # Custom fields result['Genre'] = self._parse_genres(game) return result
def _parse_search_results(self, response): """ response is expected to be a JSON object """ log.debug("Parsing response for search results: {0}".format(response)) results = [] if response['number_of_total_results'] == 0: log.warn("No results found") return results for result in response['results']: try: year = self._parse_date(result['release_date']) results.append({ 'id': result['guid'], 'title': result['name'], 'releaseDate': year, 'SearchKey': [result['name']] }) except KeyError: log.warn("Unable to find expected field in response") except Exception as e: log.warn("Error parsing field: {0}".format(e)) log.debug("Found {0} results using requests JSON parser: {1}".format( len(results), results)) return results
def _parseGameResult(self, game): result = {} # Standard fields for k, v in self._game_mapping.items(): # HACK - This is only used to retain backwards compatibility with existing scraper, where each key value was a # list, even if only one value is in that list try: result[k] = [game[v]] except Exception: # Typically this result doesn't have this field log.debug("Unable to extract data from key {0}".format(k)) # Custom fields # Adjust the date releaseDate = game['release_date'] if releaseDate is not None: result['ReleaseYear'] = [self._parse_date(releaseDate)] result['Genre'] = self._parse_lookup_data( game['genres'], self.genres['data']['genres']) result['Developer'] = self._parse_lookup_data( game['developers'], self.developers['data']['developers']) result['Publisher'] = self._parse_lookup_data( game['publishers'], self.publishers['data']['publishers']) """ # Prefix images with base url for image in ['fanart', 'boxfront', 'boxback', 'screenshot', 'clearlogo']: try: result['Filetype' + image] = ["http://thegamesdb.net/banners/" + result['Filetype' + image][0]] except KeyError: log.warn("Image type {0} not present in retrieve results".format(image)) """ return result
def add_genres_to_db(self, genreIds, gameId): # If the genre-game link doesn't exist in the DB, create it for genreId in genreIds: genreGame = GenreGame(self.gdb).getGenreGameByGenreIdAndGameId(genreId, gameId) if genreGame is None: log.debug("Inserting link between game %s and genre %s" % (str(gameId), str(genreId))) GenreGame(self.gdb).insert((genreId, gameId)) del genreGame
def __init__(self): self.env = (os.environ.get("OS", "win32"), "win32",)[os.environ.get("OS", "win32") == "xbox"] log.debug("Running environment detected as {0}".format(self.env)) # Do we need to escape commands before executing? self.escapeCmd = __addon__.getSetting(util.SETTING_RCB_ESCAPECOMMAND).upper() == 'TRUE' self.romCollection = None
def promptRomPath(self, consolename): """ Prompt the user to browse to the rompath """ dialog = xbmcgui.Dialog() # http://kodi.wiki/view/Add-on_unicode_paths romPath = dialog.browse(0, util.localize(32180) % consolename, 'files').decode('utf-8') log.debug(u"rompath selected: {0}".format(romPath)) return romPath
def __init__(self): self.env = (os.environ.get("OS", "win32"), "win32", )[os.environ.get("OS", "win32") == "xbox"] log.debug("Running environment detected as {0}".format(self.env)) # Do we need to escape commands before executing? self.escapeCmd = __addon__.getSetting(util.SETTING_RCB_ESCAPECOMMAND).upper() == 'TRUE' self.romCollection = None
def promptArtworkPath(self, console, startingDirectory): """ Prompt the user to browse to the artwork path """ dialog = xbmcgui.Dialog() # http://kodi.wiki/view/Add-on_unicode_paths artworkPath = dialog.browse(0, util.localize(32193) % console, 'files', '', False, False, startingDirectory).decode('utf-8') log.debug(u"artworkPath selected: {0}".format(artworkPath)) return artworkPath
def open_xml_url(self, **kwargs): log.info('Retrieving url %s, params = %s' %(kwargs['url'], kwargs['params'])) r = requests.get(kwargs['url'], headers=self._headers, params=kwargs['params']) log.debug(u"Retrieving {0} as XML - HTTP{1}".format(r.url, r.status_code)) # Need to ensure we are sending back Unicode text return r.text.encode('utf-8')
def insert(self, args): paramsString = ("?, " * len(args)) paramsString = paramsString[0:len(paramsString) - 2] insertString = "Insert INTO %(tablename)s VALUES (NULL, %(args)s)" % {'tablename': self.tableName, 'args': paramsString} self.gdb.cursor.execute(insertString, args) if self.gdb.cursor.rowcount == 1: log.debug("inserted values " + str(args) + self.tableName) else: log.warn("failed to insert values " + str(args) + self.tableName)
def get_launcher_by_gameid(self, gameid): """Returns the launcher class based on romCollection.useBuiltinEmulator Args: gameid: the id of the game we want to launch """ log.info("AbstractLauncher.get_launcher_by_gameid()") self.gameRow = GameView(self.gdb).getObjectById(gameid) if self.gameRow is None: log.error("Game with id %s could not be found in database" % gameid) return None try: self.romCollection = self.config.romCollections[str( self.gameRow[GameView.COL_romCollectionId])] except KeyError: log.error("Cannot get rom collection with id: " + str(self.gameRow[GameView.COL_romCollectionId])) self.gui.writeMsg(util.localize(32034)) return self.gui.writeMsg( util.localize(32163) + " " + self.gameRow[DataBaseObject.COL_NAME]) launchername = self.CMD_LAUNCHER if (self.romCollection.useBuiltinEmulator): launchername = self.RETROPLAYER_LAUNCHER #check if we already have instantiated this launcher instance = None try: instance = self._instantiated_launcher[launchername] log.debug( "Using previously instantiated launcher class {0}".format( launchername)) except KeyError: pass if not instance: log.debug("Instantiating launcher class {0}".format(launchername)) try: module = __import__(launchername.lower()) class_ = getattr(module, launchername) instance = class_() self._instantiated_launcher[launchername] = instance except ImportError: log.error("Unable to find launcher {0}".format(launchername)) raise instance.gdb = self.gdb instance.gui = self.gui instance.config = self.config return instance
def open_xml_url(self, **kwargs): log.info('Retrieving url %s, params = %s' % (kwargs['url'], kwargs['params'])) r = requests.get(kwargs['url'], headers=self._headers, params=kwargs['params']) log.debug(u"Retrieving {0} as XML - HTTP{1}".format( r.url, r.status_code)) # Need to ensure we are sending back Unicode text return r.text.encode('utf-8')
def _parseGameResult(self, response): # FIXME TODO This currently is not fully implemented result = {} if sys.version_info >= (2, 7): parser = ET.XMLParser(encoding='utf-8') else: parser = ET.XMLParser() tree = ET.fromstring(response, parser) game = tree.find('Game') # FIXME TODO others # Standard fields for k, v in self._game_mapping.items(): # HACK - This is only used to retain backwards compatibility with existing scraper, where each key value was a # list, even if only one value is in that list try: result[k] = [game.find(v).text] # FIXME TODO When we remove the hack, this will be the code to use: # result[k] = game.find(v).text except Exception: # Typically this result doesn't have this field log.debug("Unable to extract data from key {0}".format(k)) # Custom fields result['Genre'] = self._parse_genres(game.find("Genres")) # Adjust the date releaseDate = game.find("ReleaseDate") if releaseDate is not None: result['ReleaseYear'] = [self._parse_date(releaseDate.text)] # Prefix images with base url for image in [ 'fanart', 'boxfront', 'boxback', 'screenshot', 'clearlogo' ]: try: result['Filetype' + image] = [ "http://legacy.thegamesdb.net/banners/" + result['Filetype' + image][0] ] except KeyError: log.warn( "Image type {0} not present in retrieve results".format( image)) print u"Found game using ElementTree parser: {0}".format(result) return result
def extract_archive(self, rom_collection, rom, emu_params): log.info("ArchiveHandler.extract_archive") temp_dir = self.__get_temp_dir_path(rom_collection.name) self.__delete_temp_files(temp_dir) try: files_in_archive = self.__get_rom_names_from_archive(rom) except Exception as exc: log.error("Error handling compressed file: " + str(exc)) return [] if files_in_archive is None or len(files_in_archive) == 0: log.error("Error handling compressed file") return [] roms = self.__handle_indexed_roms(rom_collection.diskPrefix, files_in_archive, emu_params, rom, temp_dir) if roms: return roms try: # Extract all files to %TMP% extracted_files = self.__extract_files(rom, files_in_archive, temp_dir) except Exception as exc: log.error("Error handling compressed file: " + str(exc)) return [] if extracted_files is None: log.warn("Error handling compressed file") return [] chosen_rom = 0 if len(files_in_archive) > 1: log.info("The Archive has %d files" % len(files_in_archive)) chosen_rom = xbmcgui.Dialog().select('Choose a ROM', files_in_archive) # Point file name to the chosen file and continue as usual roms = [os.path.join(temp_dir, files_in_archive[chosen_rom])] log.debug("roms decompressed = " + str(roms)) if len(roms) == 0: return [] return roms
def checkGameHasSaveStates(self, romCollection, gameRow, filenameRows): log.info("AbstractLauncher.checkGameHasSaveStates()") stateFile = '' saveStateParams = '' if romCollection.saveStatePath == '': log.debug("No save state path set") return '' rom = filenameRows[0][0] saveStatePath = self.replacePlaceholdersInParams( romCollection.saveStatePath, rom, gameRow) saveStateFiles = glob.glob(saveStatePath) if len(saveStateFiles) == 0: log.debug("No save state files found") return '' log.info('saveStateFiles found: ' + str(saveStateFiles)) # don't select savestatefile if ASKNUM is requested in Params if re.search('(?i)%ASKNUM%', romCollection.saveStateParams): stateFile = saveStateFiles[0] else: options = [util.localize(32165)] for f in saveStateFiles: options.append(os.path.basename(f)) selectedFile = xbmcgui.Dialog().select(util.localize(32166), options) # If selections is canceled or "Don't launch statefile" option if selectedFile < 1: stateFile = '' else: stateFile = saveStateFiles[selectedFile - 1] if stateFile != '': saveStateParams = romCollection.saveStateParams if self.escapeCmd: stateFile = re.escape(stateFile) pattern = re.compile('%statefile%', re.IGNORECASE) saveStateParams = pattern.sub(stateFile, saveStateParams) return saveStateParams
def resolveParseResult(self, result, itemName): resultValue = u'' try: resultValue = result[itemName][0] if (isinstance(resultValue, str)): resultValue = resultValue.strip() resultValue = util.convertToUnicodeString(resultValue) except Exception as exc: log.warn(u"Error while resolving item: %s: %s" % (itemName, exc)) try: log.debug(u"Result %s = %s" % (itemName, resultValue)) except: pass return resultValue
def checkRomfileAlreadyExists(self, filename, enableFullReimport): isUpdate = False gameId = None log.debug("Checking if file already exists in DB: %s" % filename) romFile = File(self.gdb).getFileByNameAndType(filename, 0) if romFile is not None: isUpdate = True gameId = romFile[3] # FIXME TODO Replace with FILE_parentId log.info("File '%s' already exists in database." % filename) log.info("Always rescan imported games = {0}".format(enableFullReimport)) if enableFullReimport is False: log.info("Won't scrape this game again. Set 'Always rescan imported games' to True to force scraping.") return False, isUpdate, gameId else: log.debug("Couldn't find file in DB") return True, isUpdate, gameId
def _parse_search_results(self, response): results = [] """ response is expected to be a JSON object """ log.debug("Parsing response for search results: {0}".format(response)) if len(response["games"]) == 0: log.warn("No results found") return results for result in response['games']: results.append({'id': result['game_id'], 'title': result['title'], 'releaseDate': "", # MobyGames search doesn't return a year in brief mode 'SearchKey': [result['title']]}) log.debug("Found {0} results using requests JSON parser: {1}".format(len(results), results)) return results
def readImagePlacing(self, imagePlacingName, tree): #fileTypeForRow = None fileTypeForRows = tree.findall('ImagePlacing/fileTypeFor') fileTypeForRow = next( (element for element in fileTypeForRows if element.attrib.get('name') == imagePlacingName), None) if fileTypeForRow is None: Logutil.log( 'Configuration error. ImagePlacing/fileTypeFor %s does not exist in config.xml' % str(imagePlacingName), util.LOG_LEVEL_ERROR) return None, util.localize(32005) imagePlacing = ImagePlacing() imagePlacing.name = imagePlacingName for attr in [ 'fileTypesForGameList', 'fileTypesForGameListSelected', 'fileTypesForMainView1', 'fileTypesForMainView2', 'fileTypesForMainView3', 'fileTypesForMainViewBackground', 'fileTypesForMainViewGameInfoBig', 'fileTypesForMainViewGameInfoUpperLeft', 'fileTypesForMainViewGameInfoUpperRight', 'fileTypesForMainViewGameInfoLowerLeft', 'fileTypesForMainViewGameInfoLowerRight', 'fileTypesForMainViewGameInfoLower', 'fileTypesForMainViewGameInfoUpper', 'fileTypesForMainViewGameInfoRight', 'fileTypesForMainViewGameInfoLeft', 'fileTypesForMainViewVideoWindowBig', 'fileTypesForMainViewVideoWindowSmall', 'fileTypesForMainViewVideoFullscreen' ]: # Hack - class attribute fileTypesForXXX doesn't match XML key fileTypeForXXX val = self.readFileTypeForElement( fileTypeForRow, attr.replace('fileTypesFor', 'fileTypeFor'), tree) log.debug("Reading imageplacing for {0}: {1}".format(attr, val)) setattr(imagePlacing, attr, val) return imagePlacing, ''
def addNewElements(self, results, newResults): """ Add fields from the results to the existing set of results, adding if new, replacing if empty. This allows us to add fields from subsequent site scrapes that were missing or not available in previous sites. Args: results: Existing results dict from previous scrapes newResults: Result dict from most recent scrape Returns: Updated dict of result fields """ try: log.debug("Before merging results: %s vs %s" % (results.items(), newResults.items())) # Retain any existing key values that aren't an empty list, overwrite all others z = dict(newResults.items() + dict((k, v) for k, v in results.iteritems() if len(v) > 0).items()) log.debug("After merging results: %s" % z.items()) return z except Exception as e: # Return original results without doing anything log.warn("Error when merging results: %s" % e) return results
def addNewElements(self, results, newResults): """ Add fields from the results to the existing set of results, adding if new, replacing if empty. This allows us to add fields from subsequent site scrapes that were missing or not available in previous sites. Args: results: Existing results dict from previous scrapes newResults: Result dict from most recent scrape Returns: Updated dict of result fields """ try: log.debug("Before merging results: %s vs %s" % (list(results.items()), list(newResults.items()))) # Retain any existing key values that aren't an empty list, overwrite all others z = dict(list(newResults.items()) + list(dict((k, v) for k, v in list(results.items()) if len(v) > 0).items())) log.debug("After merging results: %s" % list(z.items())) return z except Exception as e: # Return original results without doing anything log.warn("Error when merging results: %s" % e) return results
def resolveParseResult(self, result, itemName): resultValue = "" try: resultValue = result[itemName][0] resultValue = util.html_unescape(resultValue) resultValue = resultValue.strip() # unescape ugly html encoding from websites resultValue = HTMLParser.HTMLParser().unescape(resultValue) except Exception as e: # log.warn("Error while resolving item: " + itemName + " : " + str(exc)) log.warn("Error while resolving item: {0} : {1} {2}".format(itemName, type(e), str(e))) try: log.debug("Result " + itemName + " = " + resultValue) except: pass return resultValue
def open_json_url(self, **kwargs): log.info('Retrieving url %s, params = %s' %(kwargs['url'], kwargs['params'])) try: r = requests.get(kwargs['url'], headers=self._headers, params=kwargs['params']) except ValueError as e: # Typically non-JSON response raise ScraperUnexpectedContentException("Non-JSON response received") log.debug(u"Retrieving {0} as JSON - HTTP{1}".format(r.url, r.status_code)) if r.status_code == 401: # Mobygames and GiantBomb send a 401 if the API key is invalid raise ScraperUnauthorisedException("Invalid API key sent") if r.status_code == 429: raise ScraperExceededAPIQuoteException("Scraper exceeded API key limits") if r.status_code == 500: raise ScraperWebsiteUnavailableException("Website unavailable") return r.json()
def __copyLauncherScriptsToUserdata(self): log.info('__copyLauncherScriptsToUserdata') oldBasePath = os.path.join(util.getAddonInstallPath(), 'resources', 'scriptfiles') newBasePath = os.path.join(util.getAddonDataPath(), 'scriptfiles') files = [] # Copy applaunch shell script/batch file if self.env == 'win32': files.append('applaunch.bat') else: files.append('applaunch.sh') # Copy VBS files if self.env == 'win32' and __addon__.getSetting(util.SETTING_RCB_USEVBINSOLOMODE).lower() == 'true': files += ['applaunch-vbs.bat', 'LaunchKodi.vbs', 'Sleep.vbs'] for f in files: if not xbmcvfs.exists(os.path.join(newBasePath, f)): log.debug("Copying file {0} from {1} to {2}".format(f, oldBasePath, newBasePath)) if not xbmcvfs.copy(os.path.join(oldBasePath, f), os.path.join(newBasePath, f)): log.warn("Error copying file")
def getArtworkForGame(self, romCollection, gamename, gamenameFromFile, gamedescription, foldername, publisher, developer): artWorkFound = False artworkfiles = {} artworkurls = {} for path in romCollection.mediaPaths: log.info("FileType: {0}".format(path.fileType.name)) # TODO replace %ROMCOLLECTION%, %PUBLISHER%, ... fileName = path.path.replace("%GAME%", gamenameFromFile) continueUpdate, artworkurls = self.getThumbFromOnlineSource(gamedescription, path.fileType.name, fileName, artworkurls) if not continueUpdate: return False, {}, {} log.debug("Additional data path: %s" % path.path) files = self.resolvePath((path.path,), gamename, gamenameFromFile, foldername, romCollection.name, publisher, developer) if len(files) > 0: artWorkFound = True # HACK: disable static image check as a preparation for new default image handling (this code has problems with [] in rom names) """ imagePath = str(self.resolvePath((path.path,), gamename, gamenameFromFile, foldername, romCollection.name, publisher, developer)) staticImageCheck = imagePath.upper().find(gamenameFromFile.upper()) #make sure that it was no default image that was found here if(staticImageCheck != -1): artWorkFound = True """ else: self.missingArtworkFile.add_entry(gamename, gamenameFromFile, path.fileType.name) artworkfiles[path.fileType] = files return artWorkFound, artworkfiles, artworkurls
def _parse_search_results(self, response): """ response is expected to be a JSON object """ log.debug("Parsing response for search results: {0}".format(response)) results = [] if response['number_of_total_results'] == 0: log.warn("No results found") return results for result in response['results']: try: year = self._parse_date(result['release_date']) results.append({'id': result['guid'], 'title': result['name'], 'releaseDate': year, 'SearchKey': [result['name']]}) except KeyError as k: log.warn("Unable to find expected field in response") except Exception as e: log.warn("Error parsing field: {0}".format(e)) log.debug("Found {0} results using requests JSON parser: {1}".format(len(results), results)) return results
def __postDelay(self): postDelay = __addon__.getSetting(SETTING_RCB_POSTLAUNCHDELAY) if postDelay != '': log.debug("Post delaying by {0}ms".format(postDelay)) xbmc.sleep(int(float(postDelay)))
def getThumbFromOnlineSource(self, gamedescription, fileType, fileName, artworkurls): log.info("Get thumb from online source") try: # maybe we got a thumb url from desc parser thumbKey = 'Filetype' + fileType log.info("using key: %s" % thumbKey) thumbUrl = self.resolveParseResult(gamedescription, thumbKey) if thumbUrl == '': return True, artworkurls artworkurls[fileType] = thumbUrl log.info("Get thumb from url: %s" % thumbUrl) rootExtFile = os.path.splitext(fileName) rootExtUrl = os.path.splitext(thumbUrl) files = [] if len(rootExtUrl) == 2 and len(rootExtFile) != 0: fileName = rootExtFile[0] + rootExtUrl[1] gameName = rootExtFile[0] + ".*" files = self.getFilesByWildcard(gameName) del rootExtFile, rootExtUrl if len(files) > 0: log.info("File already exists. Won't download again.") return True, artworkurls # Create folder if it doesn't already exist dirname = os.path.join(os.path.dirname(fileName), '') # Add the trailing slash that xbmcvfs.exists expects log.debug("Checking for artwork directory %s" % dirname) if KodiVersions.getKodiVersion() >= KodiVersions.KRYPTON: exists = xbmcvfs.exists(dirname) else: exists = os.path.exists(dirname) if not exists: log.info("Artwork directory %s doesn't exist, creating it" % dirname) success = xbmcvfs.mkdirs(dirname) log.info("Directory successfully created: %s" %success) if not success: #HACK: check if directory was really not created. directoryExists = xbmcvfs.exists(dirname) log.info("Directory exists: %s" %directoryExists) if not directoryExists: log.error("Could not create artwork directory: '%s'" % dirname) xbmcgui.Dialog().ok(util.localize(32010), util.localize(32011)) del dirname return False, artworkurls log.info("File %s does not exist, starting download" % fileName) # Dialog Status Art Download # Update progress dialog to state we are downloading art try: msg = "%s: %s" % (util.localize(32123), self._guiDict["gameNameKey"]) submsg = "%s - downloading art" % self._guiDict["scraperSiteKey"][thumbKey] self._gui.writeMsg(self._guiDict["dialogHeaderKey"], msg, submsg, self._guiDict["fileCountKey"]) except KeyError: log.warn("Unable to retrieve key from GUI dict") try: self.download_thumb(thumbUrl, fileName) except Exception, (exc): log.error("Could not create file: '%s'. Error message: '%s'" % (fileName, exc)) # xbmcgui.Dialog().ok(util.localize(32012), util.localize(32011)) return False, artworkurls Logutil.log("Download finished.", util.LOG_LEVEL_INFO)
def __buildCmd(self, gui, filenameRows, gameRow, calledFromSkin): log.info("launcher.buildCmd") compressedExtensions = ['7z', 'zip'] cmd = "" precmd = "" postcmd = "" emuCommandLine = self.romCollection.emulatorCmd log.info("emuCommandLine: " + emuCommandLine) log.info("preCmdLine: " + self.romCollection.preCmd) log.info("postCmdLine: " + self.romCollection.postCmd) # handle savestates stateFile = self.__checkGameHasSaveStates(gameRow, filenameRows) if stateFile == '': emuParams = self.romCollection.emulatorParams else: emuParams = self.romCollection.saveStateParams if self.escapeCmd: stateFile = re.escape(stateFile) emuParams = emuParams.replace('%statefile%', stateFile) emuParams = emuParams.replace('%STATEFILE%', stateFile) emuParams = emuParams.replace('%Statefile%', stateFile) # params could be: {-%I% %ROM%} # we have to repeat the part inside the brackets and replace the %I% with the current index emuParams, partToRepeat = self.__prepareMultiRomCommand(emuParams) # ask for disc number if multidisc game diskName = "" if self.romCollection.diskPrefix != '' and '%I%' not in emuParams: log.info("Getting Multiple Disc Parameter") options = [] for disk in filenameRows: gamename = os.path.basename(disk[0]) match = re.search(self.romCollection.diskPrefix.lower(), str(gamename).lower()) if match: disk = gamename[match.start():match.end()] options.append(disk) if len(options) > 1 and not calledFromSkin: diskNum = xbmcgui.Dialog().select(util.localize(32164) + ': ', options) if diskNum < 0: # don't launch game log.info("No disc was chosen. Won't launch game") return "", "", "", None else: diskName = options[diskNum] log.info("Chosen Disc: %s" % diskName) # insert game specific command gameCmd = '' if gameRow[util.GAME_gameCmd] is not None: gameCmd = str(gameRow[util.GAME_gameCmd]) # be case insensitive with (?i) emuParams = re.sub('(?i)%gamecmd%', gameCmd, emuParams) log.info("emuParams: " + emuParams) fileindex = int(0) for fileNameRow in filenameRows: rom = fileNameRow[0] log.info("rom: " + str(rom)) if self.romCollection.makeLocalCopy: localDir = os.path.join(util.getTempDir(), self.romCollection.name) if xbmcvfs.exists(localDir + '\\'): log.info("Trying to delete local rom files") dirs, files = xbmcvfs.listdir(localDir) for f in files: xbmcvfs.delete(os.path.join(localDir, f)) localRom = os.path.join(localDir, os.path.basename(str(rom))) log.info("Creating local copy: " + str(localRom)) if xbmcvfs.copy(rom, localRom): log.info("Local copy created") rom = localRom # If it's a .7z file # Don't extract zip files in case of savestate handling and when called From skin filext = rom.split('.')[-1] roms = [rom] if filext in compressedExtensions and not self.romCollection.doNotExtractZipFiles and stateFile == '' and not calledFromSkin: roms = self.__handleCompressedFile(gui, filext, rom, emuParams) log.debug("roms compressed = " + str(roms)) if len(roms) == 0: return "", "", "", None # no use for complete cmd as we just need the game name if self.romCollection.useBuiltinEmulator: log.debug("roms = " + str(roms)) return "", "", "", roms del rom for rom in roms: precmd = "" postcmd = "" if fileindex == 0: emuParams = self.__replacePlaceholdersInParams(emuParams, rom, gameRow) if self.escapeCmd: emuCommandLine = re.escape(emuCommandLine) if self.romCollection.name in ['Linux', 'Macintosh', 'Windows']: cmd = self.__replacePlaceholdersInParams(emuCommandLine, rom, gameRow) else: cmd = '\"' + emuCommandLine + '\" ' + emuParams.replace('%I%', str(fileindex)) else: newrepl = partToRepeat newrepl = self.__replacePlaceholdersInParams(newrepl, rom, gameRow) if self.escapeCmd: emuCommandLine = re.escape(emuCommandLine) newrepl = newrepl.replace('%I%', str(fileindex)) cmd += ' ' + newrepl cmdprefix = '' if self.env == "win32": cmdprefix = 'call ' precmd = cmdprefix + self.__replacePlaceholdersInParams(self.romCollection.preCmd, rom, gameRow) postcmd = cmdprefix + self.__replacePlaceholdersInParams(self.romCollection.postCmd, rom, gameRow) fileindex += 1 # A disk was chosen by the user, select it here if diskName: log.info("Choosing Disk: " + str(diskName)) match = re.search(self.romCollection.diskPrefix.lower(), cmd.lower()) replString = cmd[match.start():match.end()] cmd = cmd.replace(replString, diskName) return cmd, precmd, postcmd, roms
def useSingleScrapers(self, romCollection, romFile, gamenameFromFile, progDialogRCHeader, fileCount): """Scrape site for game metadata Args: romCollection: gamenameFromFile: progDialogRCHeader: fileCount: Returns: dict for the game result: {'SearchKey': ['Chrono Trigger'], 'Publisher': ['Squaresoft'], 'Description': ["The millennium. A portal is opened. The chain of time is broken...], 'Players': ['1'], 'Platform': ['Super Nintendo (SNES)'], 'Game': ['Chrono Trigger'], 'Filetypeboxfront': ['http://thegamesdb.net/banners/boxart/original/front/1255-1.jpg'], 'Filetypeboxback': ['http://thegamesdb.net/banners/boxart/original/back/1255-1.jpg'], 'Filetypescreenshot': ['http://thegamesdb.net/banners/screenshots/1255-1.jpg', 'http://thegamesdb.net/banners/screenshots/1255-2.jpg', 'http://thegamesdb.net/banners/screenshots/1255-3.jpg', 'http://thegamesdb.net/banners/screenshots/1255-4.jpg', 'http://thegamesdb.net/banners/screenshots/1255-5.jpg'], 'Filetypefanart': ['http://thegamesdb.net/banners/fanart/original/1255-1.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-10.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-11.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-2.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-3.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-4.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-5.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-6.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-7.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-8.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-9.jpg'], 'Genre': ['Role-Playing'], 'Developer': ['Squaresoft']} dict for artwork urls: {'Filetypefanart': 'thegamesdb.net', 'Filetypeboxback': 'thegamesdb.net', 'Filetypescreenshot': 'thegamesdb.net', 'Filetypeboxfront': 'thegamesdb.net'} Note - this only contains entries for artwork that was found (i.e. is not empty list) """ gameresult = {} artScrapers = {} scraperSite = romCollection.scraperSites[0] try: #first check if a local nfo file is available nfoscraper = NFO_Scraper() nfofile = nfoscraper.get_nfo_path(gamenameFromFile, romCollection.name, romFile) if xbmcvfs.exists(nfofile) and __addon__.getSetting(util.SETTING_RCB_PREFERLOCALNFO).upper() == 'TRUE': log.info("Found local nfo file. Using this to scrape info.") newscraper = nfoscraper else: newscraper = AbstractScraper().get_scraper_by_name(scraperSite.name) #set path to desc file (only useful for offline scrapers) newscraper.path = scraperSite.path results = newscraper.search(gamenameFromFile, romCollection.name) log.debug(u"Searching for %s - found %s results: %s" % (gamenameFromFile, len(results), results)) except ScraperExceededAPIQuoteException as ke: # API key is invalid - we need to stop scraping log.error("Scraper exceeded API key, stopping scraping") raise except Exception as e: log.error("Error searching for %s using scraper %s - %s %s" % ( gamenameFromFile, scraperSite.name, type(e), e)) return gameresult, artScrapers if results == []: log.warn("No search results found for %s using scraper %s" % (gamenameFromFile, scraperSite.name)) return gameresult, artScrapers matched = Matcher().getBestResults(results, gamenameFromFile) if matched is None: log.error("No matches found for %s, skipping" % gamenameFromFile) return gameresult, artScrapers log.debug("After matching: %s" % matched) try: retrievedresult = newscraper.retrieve(matched['id'], romCollection.name) log.debug(u"Retrieving %s - found %s" % (matched['id'], retrievedresult)) except Exception as e: # FIXME TODO Catch exceptions specifically log.error("Error retrieving %s - %s %s" % (matched['id'], type(e), e)) return gameresult, artScrapers # Update the gameresult with any new fields gameresult = self.addNewElements(gameresult, retrievedresult) self._gui.writeMsg(progDialogRCHeader, util.localize(32123) + ": " + gamenameFromFile, scraperSite.name + " - " + util.localize(32131), fileCount) # Find Filetypes and Scrapers for Art Download # FIXME TODO The following is kept to keep artwork downloading working as it currently is. We already have # the URLs and so could handle/download here, rather than deferring if len(gameresult) > 0: for path in romCollection.mediaPaths: thumbKey = 'Filetype' + path.fileType.name if len(self.resolveParseResult(gameresult, thumbKey)) > 0: if (thumbKey in artScrapers) == 0: artScrapers[thumbKey] = scraperSite.name log.debug(u"After scraping, result = %s, artscrapers = %s" % (gameresult, artScrapers)) return gameresult, artScrapers
def __preDelay(self): preDelay = __addon__.getSetting(SETTING_RCB_PRELAUNCHDELAY) if preDelay != '': log.debug("Pre delaying by {0}ms".format(preDelay)) xbmc.sleep(int(float(preDelay)))
def resolveParseResult(self, result, itemName): resultValue = u'' try: resultValue = result[itemName][0].strip() if type(resultValue) == str: resultValue = resultValue.decode('utf-8') except Exception, (exc): log.warn(u"Error while resolving item: %s: %s" % (itemName, exc)) try: log.debug(u"Result %s = %s" % (itemName, resultValue)) except: pass return resultValue def insertFile(self, fileName, gameId, fileType, romCollectionId, publisherId, developerId): log.debug("Begin Insert file: %s" % fileName) parentId = None # TODO console and romcollection could be done only once per RomCollection # fileTypeRow[3] = parent if fileType.parent == 'game': parentId = gameId elif fileType.parent == 'romcollection':
def __audioSuspend(self): if __addon__.getSetting(util.SETTING_RCB_SUSPENDAUDIO).upper() == 'TRUE': log.debug("Suspending audio") xbmc.executebuiltin("PlayerControl(Stop)") xbmc.enableNavSounds(False) xbmc.audioSuspend()
def __audioResume(self): if __addon__.getSetting(util.SETTING_RCB_SUSPENDAUDIO).upper() == 'TRUE': log.debug("Resuming audio") xbmc.audioResume() xbmc.enableNavSounds(True)