def __getArchives7z(self, filepath, archiveList, directory_to):
try:
import py7zlib
except ImportError:
# 32039 = Error launching .7z file.
# 32129 = Please check kodi.log for details.
message = "%s[CR]%s" % (util.localize(32039), util.localize(32129))
xbmcgui.Dialog().ok(util.SCRIPTNAME, message)
msg = (
"You have tried to launch a .7z file but you are missing required libraries to extract the file. "
"You can download the latest RCB version from RCBs project page. It contains all required libraries."
)
log.error(msg)
return None
fp = open(str(filepath), 'rb')
archive = py7zlib.Archive7z(fp)
archivesDecompressed = [(name, archive.getmember(name).read())
for name in archiveList]
fp.close()
if archivesDecompressed is None:
log.warn("Error handling compressed file")
return []
for archive in archivesDecompressed:
newPath = os.path.join(directory_to, archive[0])
log.info("Putting extracted file in %s" % newPath)
fo = open(str(newPath), 'wb')
fo.write(archive[1])
fo.close()
return archivesDecompressed
def __copyLauncherScriptsToUserdata(self):
log.info('__copyLauncherScriptsToUserdata')
oldBasePath = os.path.join(util.getAddonInstallPath(), 'resources',
'scriptfiles')
newBasePath = os.path.join(util.getAddonDataPath(), 'scriptfiles')
files = []
# Copy applaunch shell script/batch file
if self.env == 'win32':
files.append('applaunch.bat')
else:
files.append('applaunch.sh')
# Copy VBS files
if self.env == 'win32' and __addon__.getSetting(
util.SETTING_RCB_USEVBINSOLOMODE).lower() == 'true':
files += ['applaunch-vbs.bat', 'LaunchKodi.vbs', 'Sleep.vbs']
for f in files:
if not xbmcvfs.exists(os.path.join(newBasePath, f)):
log.debug("Copying file {0} from {1} to {2}".format(
f, oldBasePath, newBasePath))
if not xbmcvfs.copy(os.path.join(oldBasePath, f),
os.path.join(newBasePath, f)):
log.warn("Error copying file")
def insertFile(self, fileName, gameId, fileType, romCollectionId, publisherId, developerId):
log.debug("Begin Insert file: %s" % fileName)
parentId = None
# TODO console and romcollection could be done only once per RomCollection
# fileTypeRow[3] = parent
if fileType.parent == 'game':
parentId = gameId
elif fileType.parent == 'romcollection':
parentId = romCollectionId
elif fileType.parent == 'publisher':
parentId = publisherId
elif fileType.parent == 'developer':
parentId = developerId
log.info("Inserting file with parent {0} (type {1})".format(parentId, fileType.parent))
fileRow = File(self.gdb).getFileByNameAndTypeAndParent(fileName, fileType.id, parentId)
if fileRow is None:
log.info("File does not exist in database. Insert file: %s" % fileName)
f = File(self.gdb)
try:
f.insert((fileName, fileType.id, parentId))
except Exception:
log.warn("Error inserting into database: %s" % fileName)
else:
log.info("File already exists in database: %s" % fileName)
def insertFile(self, fileName, gameId, fileType, romCollectionId, publisherId, developerId):
log.debug("Begin Insert file: %s" % fileName)
parentId = None
# TODO console and romcollection could be done only once per RomCollection
# fileTypeRow[3] = parent
if fileType.parent == 'game':
parentId = gameId
elif fileType.parent == 'romcollection':
parentId = romCollectionId
elif fileType.parent == 'publisher':
parentId = publisherId
elif fileType.parent == 'developer':
parentId = developerId
log.info("Inserting file with parent {0} (type {1})".format(parentId, fileType.parent))
fileRow = File(self.gdb).getFileByNameAndTypeAndParent(fileName, fileType.id, parentId)
if fileRow is None:
log.info("File does not exist in database. Insert file: %s" % fileName)
f = File(self.gdb)
try:
f.insert((fileName, fileType.id, parentId))
except Exception, (exc):
log.warn("Error inserting into database: %s" % fileName)
def _parse_game_result(self, response):
""" response is expected to be a JSON object """
result = {}
# Standard fields
for k, v in self._game_mapping.items():
try:
# HACK - for compatibility we need to put each result in an array
#result[k] = response[v]
result[k] = [response[v]]
except KeyError as k:
log.warn("Unable to find key: {0}".format(k))
except Exception as e:
log.warn("Unable to extract data from key {0}".format(e))
# Custom fields (i.e. ones that require special handling
# HACK - for compatibility we need to put each result in an array
result['ReleaseYear'] = [self._parse_date(response['original_release_date'])]
result['Developer'] = self._parse_developers(response['developers'])
result['Publisher'] = self._parse_publishers(response['publishers'])
result['Genre'] = self._parse_genres(response['genres'])
# FIXME TODO Artwork and images are quite cumbersome to get from giantbomb search results
return result
def insertGameFromDesc(self, gamedescription, gamename, romCollection,
filenamelist, foldername, isUpdate, gameId):
log.info("insertGameFromDesc")
if gamedescription is not None:
game = self.resolveParseResult(gamedescription, 'Game')
else:
game = ''
# if no game name has been scraped we expect that no results have been found
if game == '':
self.missingDescFile.add_entry(gamename)
if __addon__.getSetting(
util.SETTING_RCB_IGNOREGAMEWITHOUTDESC).upper() == 'TRUE':
log.warn(
"No description found for game '%s'. Game will not be imported."
% gamename)
return None
gamedescription = {}
gameId = self.insertData(gamedescription, gamename, romCollection,
filenamelist, foldername, isUpdate, gameId)
return gameId
def _parse_game_result(self, response):
""" response is expected to be a JSON object """
result = {}
# Standard fields
for k, v in self._game_mapping.items():
try:
# HACK - for compatibility we need to put each result in an array
#result[k] = response[v]
result[k] = [response[v]]
except KeyError as k:
log.warn("Unable to find key: {0}".format(k))
except Exception as e:
log.warn("Unable to extract data from key {0}".format(e))
# Custom fields (i.e. ones that require special handling
# HACK - for compatibility we need to put each result in an array
result['ReleaseYear'] = [
self._parse_date(response['original_release_date'])
]
result['Developer'] = self._parse_developers(response['developers'])
result['Publisher'] = self._parse_publishers(response['publishers'])
result['Genre'] = self._parse_genres(response['genres'])
# FIXME TODO Artwork and images are quite cumbersome to get from giantbomb search results
return result
def resolvePath(self, paths, gamename, gamenameFromFile, foldername, romCollectionName, publisher, developer):
resolvedFiles = []
for path in paths:
files = []
log.info("resolve path: %s" % path)
if path.find("%GAME%") > -1:
pathnameFromFile = path.replace("%GAME%", gamenameFromFile)
log.info("resolved path from rom file name: %s" % pathnameFromFile)
files = self.getFilesByWildcard(pathnameFromFile)
if len(files) == 0:
files = self.getFilesByGameNameIgnoreCase(pathnameFromFile)
if gamename != gamenameFromFile and len(files) == 0:
pathnameFromGameName = path.replace("%GAME%", gamename)
log.info("resolved path from game name: %s" % pathnameFromGameName)
files = self.getFilesByWildcard(pathnameFromGameName)
if len(files) == 0:
files = self.getFilesByGameNameIgnoreCase(pathnameFromGameName)
if gamename != foldername and len(files) == 0:
pathnameFromFolder = path.replace("%GAME%", foldername)
log.info("resolved path from rom folder name: %s" % pathnameFromFolder)
files = self.getFilesByWildcard(pathnameFromFolder)
if len(files) == 0:
files = self.getFilesByGameNameIgnoreCase(pathnameFromFolder)
# ODO could be done only once per RomCollection
if path.find("%ROMCOLLECTION%") > -1 and romCollectionName is not None and len(files) == 0:
pathnameFromRomCollection = path.replace("%ROMCOLLECTION%", romCollectionName)
log.info("resolved path from rom collection name: {0}".format(pathnameFromRomCollection))
files = self.getFilesByWildcard(pathnameFromRomCollection)
if path.find("%PUBLISHER%") > -1 and publisher is not None and len(files) == 0:
pathnameFromPublisher = path.replace("%PUBLISHER%", publisher)
log.info("resolved path from publisher name: %s" % pathnameFromPublisher)
files = self.getFilesByWildcard(pathnameFromPublisher)
if path.find("%DEVELOPER%") > -1 and developer is not None and len(files) == 0:
pathnameFromDeveloper = path.replace("%DEVELOPER%", developer)
log.info("resolved path from developer name: %s" % pathnameFromDeveloper)
files = self.getFilesByWildcard(pathnameFromDeveloper)
if path.find("%GAME%") == -1 & path.find("%ROMCOLLECTION%") == -1 & path.find(
"%PUBLISHER%") == -1 & path.find("%DEVELOPER%") == -1:
pathnameFromStaticFile = path
log.info("using static defined media file from path: %s" % pathnameFromStaticFile)
files = self.getFilesByWildcard(pathnameFromStaticFile)
if len(files) == 0:
log.warn("No files found for game '%s' at path '%s'. Make sure that file names are matching." % (
gamename, path))
for f in files:
if xbmcvfs.exists(f):
resolvedFiles.append(f)
return resolvedFiles
def resolvePath(self, paths, gamename, gamenameFromFile, foldername, romCollectionName, publisher, developer):
resolvedFiles = []
for path in paths:
files = []
log.info("resolve path: %s" % path)
if path.find("%GAME%") > -1:
pathnameFromFile = path.replace("%GAME%", gamenameFromFile)
log.info("resolved path from rom file name: %s" % pathnameFromFile)
files = self.getFilesByWildcard(pathnameFromFile)
if len(files) == 0:
files = self.getFilesByGameNameIgnoreCase(pathnameFromFile)
if gamename != gamenameFromFile and len(files) == 0:
pathnameFromGameName = path.replace("%GAME%", gamename)
log.info("resolved path from game name: %s" % pathnameFromGameName)
files = self.getFilesByWildcard(pathnameFromGameName)
if len(files) == 0:
files = self.getFilesByGameNameIgnoreCase(pathnameFromGameName)
if gamename != foldername and len(files) == 0:
pathnameFromFolder = path.replace("%GAME%", foldername)
log.info("resolved path from rom folder name: %s" % pathnameFromFolder)
files = self.getFilesByWildcard(pathnameFromFolder)
if len(files) == 0:
files = self.getFilesByGameNameIgnoreCase(pathnameFromFolder)
# ODO could be done only once per RomCollection
if path.find("%ROMCOLLECTION%") > -1 and romCollectionName is not None and len(files) == 0:
pathnameFromRomCollection = path.replace("%ROMCOLLECTION%", romCollectionName)
log.info("resolved path from rom collection name: {0}".format(pathnameFromRomCollection))
files = self.getFilesByWildcard(pathnameFromRomCollection)
if path.find("%PUBLISHER%") > -1 and publisher is not None and len(files) == 0:
pathnameFromPublisher = path.replace("%PUBLISHER%", publisher)
log.info("resolved path from publisher name: %s" % pathnameFromPublisher)
files = self.getFilesByWildcard(pathnameFromPublisher)
if path.find("%DEVELOPER%") > -1 and developer is not None and len(files) == 0:
pathnameFromDeveloper = path.replace("%DEVELOPER%", developer)
log.info("resolved path from developer name: %s" % pathnameFromDeveloper)
files = self.getFilesByWildcard(pathnameFromDeveloper)
if path.find("%GAME%") == -1 & path.find("%ROMCOLLECTION%") == -1 & path.find(
"%PUBLISHER%") == -1 & path.find("%DEVELOPER%") == -1:
pathnameFromStaticFile = path
log.info("using static defined media file from path: %s" % pathnameFromStaticFile)
files = self.getFilesByWildcard(pathnameFromStaticFile)
if len(files) == 0:
log.warn("No files found for game '%s' at path '%s'. Make sure that file names are matching." % (
gamename, path))
for f in files:
if xbmcvfs.exists(f):
resolvedFiles.append(f)
return resolvedFiles
def __getEncoding(self):
# HACK: sys.getfilesystemencoding() is not supported on all systems (e.g. Android)
try:
encoding = sys.getfilesystemencoding()
except Exception as e:
log.warn("Unable to get filesystem encoding, defaulting to UTF-8")
encoding = 'utf-8'
return encoding
def __getEncoding(self):
# HACK: sys.getfilesystemencoding() is not supported on all systems (e.g. Android)
try:
encoding = sys.getfilesystemencoding()
except Exception as e:
log.warn("Unable to get filesystem encoding, defaulting to UTF-8")
encoding = 'utf-8'
return encoding
def getThumbFromOnlineSource(self, gamedescription, fileType, fileName, artworkurls):
log.info("Get thumb from online source")
try:
# maybe we got a thumb url from desc parser
thumbKey = 'Filetype' + fileType
log.info("using key: %s" % thumbKey)
thumbUrl = self.resolveParseResult(gamedescription, thumbKey)
if thumbUrl == '':
return True, artworkurls
artworkurls[fileType] = thumbUrl
log.info("Get thumb from url: %s" % thumbUrl)
rootExtFile = os.path.splitext(fileName)
rootExtUrl = os.path.splitext(thumbUrl)
files = []
if len(rootExtUrl) == 2 and len(rootExtFile) != 0:
fileName = rootExtFile[0] + rootExtUrl[1]
gameName = rootExtFile[0] + ".*"
files = self.getFilesByWildcard(gameName)
del rootExtFile, rootExtUrl
if len(files) > 0:
log.info("File already exists. Won't download again.")
return True, artworkurls
log.info("File %s does not exist, starting download" % fileName)
# Dialog Status Art Download
# Update progress dialog to state we are downloading art
try:
#32123 = Importing Game
#32210 = downloading artwork
msg = "%s: %s[CR]%s: %s" % (util.localize(32123), self._guiDict["gameNameKey"],
self._guiDict["scraperSiteKey"][thumbKey], util.localize(32210))
self._gui.writeMsg(msg, self._guiDict["fileCountKey"])
except KeyError:
log.warn("Unable to retrieve key from GUI dict")
try:
self.download_thumb(thumbUrl, fileName)
except Exception as exc:
log.error("Could not create file: '%s'. Error message: '%s'" % (fileName, exc))
# xbmcgui.Dialog().ok(util.localize(32012), util.localize(32011))
return False, artworkurls
log.info("Download finished.")
except Exception as exc:
log.warn("Error in getThumbFromOnlineSource: %s" % exc)
return True, artworkurls
def insertGameFromDesc(self, gamedescription, gamename_from_file, romCollection, romfiles, foldername, isUpdate, gameId):
log.info("insertGameFromDesc")
if gamedescription is not None:
game = self.resolveParseResult(gamedescription, 'Game')
else:
game = ''
# if no game name has been scraped we expect that no results have been found
if game == '':
self.missingDescFile.add_entry(gamename_from_file)
if __addon__.getSetting(util.SETTING_RCB_IGNOREGAMEWITHOUTDESC).upper() == 'TRUE':
log.warn("No description found for game '%s'. Game will not be imported." % gamename_from_file)
return None
gamedescription = {}
game_row = self.convert_parseresult_to_gamerow(gamedescription, gamename_from_file)
game_row[Game.COL_romCollectionId] = romCollection.id
gamename = game_row[Game.COL_NAME]
publisher = self.resolveParseResult(gamedescription, 'Publisher')
developer = self.resolveParseResult(gamedescription, 'Developer')
artWorkFound, artworkfiles, artworkurls = self.getArtworkForGame(romCollection, gamename, gamename_from_file,
gamedescription, foldername, publisher, developer)
#add artwork filenames to game_row
for filetype, filenames in list(artworkfiles.items()):
for filename in filenames:
prop = 'COL_fileType%s' % filetype.id
index = getattr(Game, prop)
game_row[index] = filename
gameId = self.insertGame(game_row, isUpdate, gameId, romCollection.allowUpdate, )
if gameId is None:
return None
genreIds = self.insertForeignKeyItemList(gamedescription, 'Genre', Genre(self.gdb))
self.add_genres_to_db(genreIds, gameId)
self.add_romfiles_to_db(romfiles, gameId)
self.gdb.commit()
# Create Nfo file with game properties
try:
# Read game from db as nfowriter expects GameView db row
gamerow = GameView(self.gdb).getGameById(gameId)
writer = NfoWriter()
writer.createNfoFromDesc(gamerow, romCollection.name, romfiles[0], gamename_from_file, artworkfiles, artworkurls)
except Exception as e:
log.warn(u"Unable to write NFO file for game %s: %s" % (gamename, e))
return gameId
def insert(self, args):
paramsString = ("?, " * len(args))
paramsString = paramsString[0:len(paramsString) - 2]
insertString = "Insert INTO %(tablename)s VALUES (NULL, %(args)s)" % {'tablename': self.tableName,
'args': paramsString}
self.gdb.cursor.execute(insertString, args)
if self.gdb.cursor.rowcount == 1:
log.debug("inserted values " + str(args) + self.tableName)
else:
log.warn("failed to insert values " + str(args) + self.tableName)
def _update_artwork_cache_for_romcollection(self, rom_collection,
file_type_id, media_dict):
log.info('Begin _update_artwork_cache_for_romcollection')
log.info('Update artwork cache for Rom Collection %s' %
str(rom_collection.id))
continue_update = True
media_paths_dict = {}
try:
media_paths_dict = media_dict[rom_collection.id]
except KeyError:
log.warn('No media paths dict found for rom collection %s' %
rom_collection.id)
return continue_update
games = GameView(self.gdb).getFilteredGames(rom_collection.id, 0, 0, 0,
0, 0, 0, 0, 0, '0 = 0', '',
0)
gamecount = 1
for game in games:
self.progress_dialog.itemCount = len(games)
#32955 = Scan artwork for Game
update_message = "%s[CR]%s: %i/%i" % (
self.dialogheader, util.localize(32955), gamecount, len(games))
continue_update = self.progress_dialog.writeMsg(
update_message, gamecount)
if not continue_update:
log.info('Update canceled')
break
gamecount = gamecount + 1
for media_path in rom_collection.mediaPaths:
#check if we should handle this file type
if str(file_type_id
) != media_path.fileType.id and file_type_id != 0:
continue
roms = File(self.gdb).getRomsByGameId(game[GameView.COL_ID])
gamename_from_file = rom_collection.getGamenameFromFilename(
roms[0][0])
#check if artwork is available for this type
file = self._find_file_in_mediadict(media_path.fileType.id,
rom_collection,
media_paths_dict,
gamename_from_file)
#check if filename has changed
if game[getattr(GameView, "COL_fileType%s" %
media_path.fileType.id)] != file:
#write result to db
# get column name from FIELDNAMES - index is the same as db column index (COL_fileTypeX)
column = Game.FIELDNAMES[getattr(
Game, "COL_fileType%s" % media_path.fileType.id)]
Game(self.gdb).update((column, ), (file, ),
game[Game.COL_ID], True)
return continue_update
def matchGamename(self, results, gamenameFromFile):
for idx, result in enumerate(results):
try:
# Check if the result has the correct platform (if needed)
found_platform = self.resolveParseResult(
result, 'PlatformSearchKey')
if found_platform != '' and self.expected_platform != found_platform:
log.info(
"Platform mismatch. %s != %s. Result will be skipped."
% (self.expected_platform, found_platform))
continue
searchKey = self.resolveParseResult(result, 'SearchKey')
# keep it for later reference
searchkey_orig = searchKey
gamename_orig = gamenameFromFile
# if no searchkey is specified first result is valid (1 file per game scenario)
if searchkey_orig == '':
log.info("No searchKey found. Using first result")
return result
log.info("Comparing %s with %s" %
(gamename_orig, searchkey_orig))
if gamename_orig == searchkey_orig:
# perfect match
return result
# normalize name and searchkey before comparison
gnu = GameNameUtil()
gamename_normalized = gnu.normalize_name(gamename_orig)
searchkey_normalized = gnu.normalize_name(searchkey_orig)
log.info("Try normalized names. Comparing %s with %s" %
(gamename_normalized, searchkey_normalized))
if gamename_normalized == searchkey_normalized:
# perfect match
return result
#strip additional info from gamename
gamename_stripped = gnu.strip_addinfo_from_name(gamename_orig)
gamename_stripped = gnu.normalize_name(gamename_stripped)
log.info(
"Try with stripped additional info. Comparing %s with %s" %
(gamename_stripped, searchkey_normalized))
if gamename_stripped == searchkey_normalized:
# perfect match
return result
except Exception as exc:
log.warn("An error occured while matching the best result: " +
str(exc))
log.info("No result found for gamename %s" % gamenameFromFile)
return None
def resolveParseResult(self, result, itemName):
resultValue = u''
try:
resultValue = result[itemName][0].strip()
if type(resultValue) == str:
resultValue = resultValue.decode('utf-8')
except Exception, (exc):
log.warn(u"Error while resolving item: %s: %s" % (itemName, exc))
def resolveParseResult(self, result, itemName):
resultValue = u''
try:
resultValue = result[itemName][0].strip()
if type(resultValue) == str:
resultValue = resultValue.decode('utf-8')
except Exception, (exc):
log.warn(u"Error while resolving item: %s: %s" % (itemName, exc))
def _parse_screenshots_result(self, response):
result = {}
screenshots = response['screenshots']
if len(screenshots) == 0:
log.warn("No screenshots found in mobygames response")
return result
#HACK: always return the first screenshot. We could add support for multiple screenshots or screenshot selection.
screenshot = screenshots[0]
result['Filetypescreenshot'] = [screenshot['image']]
return result
def _parseGameResult(self, response):
# FIXME TODO This currently is not fully implemented
result = {}
if sys.version_info >= (2, 7):
parser = ET.XMLParser(encoding='utf-8')
else:
parser = ET.XMLParser()
tree = ET.fromstring(response, parser)
game = tree.find('Game')
# FIXME TODO others
# Standard fields
for k, v in self._game_mapping.items():
# HACK - This is only used to retain backwards compatibility with existing scraper, where each key value was a
# list, even if only one value is in that list
try:
result[k] = [game.find(v).text]
# FIXME TODO When we remove the hack, this will be the code to use:
# result[k] = game.find(v).text
except Exception:
# Typically this result doesn't have this field
log.debug("Unable to extract data from key {0}".format(k))
# Custom fields
result['Genre'] = self._parse_genres(game.find("Genres"))
# Adjust the date
releaseDate = game.find("ReleaseDate")
if releaseDate is not None:
result['ReleaseYear'] = [self._parse_date(releaseDate.text)]
# Prefix images with base url
for image in [
'fanart', 'boxfront', 'boxback', 'screenshot', 'clearlogo'
]:
try:
result['Filetype' + image] = [
"http://legacy.thegamesdb.net/banners/" +
result['Filetype' + image][0]
]
except KeyError:
log.warn(
"Image type {0} not present in retrieve results".format(
image))
print u"Found game using ElementTree parser: {0}".format(result)
return result
def _parse_date(self, datestr):
"""Extract the year from a given date string using a given format. This function is used to cater for
an edge case identified in https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507.
Args:
datestr: Input date
Returns:
Year as a %Y format string
"""
if datestr is None:
return '1970'
x = None
for fmt2 in ["%Y-%m-%d", "%Y-%m", "%Y", "%Y-%m-%d %H:%M:%S", "%d/%m/%Y", "%m/%d/%Y"]:
try:
x = datetime.strptime(datestr, fmt2).strftime("%Y")
break
except ValueError as e:
# Skip to the next format
log.warn("ValueError in parseDate: %s" %e)
except TypeError as e:
log.warn("Unable to parse date using strptime, falling back to time function")
try:
x = datetime(*(time.strptime(datestr, fmt2)[0:6])).strftime("%Y")
break
except ValueError as ve:
log.warn("Unable to parse date %s using %s, try next format. %s" %(datestr, fmt2, ve))
if x is not None:
return x
else:
log.warn(u"Unexpected date format: {0}".format(datestr))
return u"1970"
def _parse_screenshots_result(self, response):
result = {}
screenshots = response['screenshots']
if len(screenshots) == 0:
log.warn("No screenshots found in mobygames response")
return result
#HACK: always return the first screenshot. We could add support for multiple screenshots or screenshot selection.
screenshot = screenshots[0]
result['Filetypescreenshot'] = [screenshot['image']]
return result
def __executeCommand(self, cmd):
# change working directory
path = os.path.dirname(self.romCollection.emulatorCmd)
if os.path.isdir(path):
try:
os.chdir(path)
except OSError:
log.warn("Unable to chdir to {0}".format(path))
if self.romCollection.usePopen:
import subprocess
process = subprocess.Popen(cmd.encode(self.__getEncoding()), shell=True)
process.wait()
else:
os.system(cmd.encode(self.__getEncoding()))
def extract_archive(self, rom_collection, rom, emu_params):
log.info("ArchiveHandler.extract_archive")
temp_dir = self.__get_temp_dir_path(rom_collection.name)
self.__delete_temp_files(temp_dir)
try:
files_in_archive = self.__get_rom_names_from_archive(rom)
except Exception as exc:
log.error("Error handling compressed file: " + str(exc))
return []
if files_in_archive is None or len(files_in_archive) == 0:
log.error("Error handling compressed file")
return []
roms = self.__handle_indexed_roms(rom_collection.diskPrefix,
files_in_archive, emu_params, rom,
temp_dir)
if roms:
return roms
try:
# Extract all files to %TMP%
extracted_files = self.__extract_files(rom, files_in_archive,
temp_dir)
except Exception as exc:
log.error("Error handling compressed file: " + str(exc))
return []
if extracted_files is None:
log.warn("Error handling compressed file")
return []
chosen_rom = 0
if len(files_in_archive) > 1:
log.info("The Archive has %d files" % len(files_in_archive))
chosen_rom = xbmcgui.Dialog().select('Choose a ROM',
files_in_archive)
# Point file name to the chosen file and continue as usual
roms = [os.path.join(temp_dir, files_in_archive[chosen_rom])]
log.debug("roms decompressed = " + str(roms))
if len(roms) == 0:
return []
return roms
def getRomCollectionById(self, id):
"""
Find the matching Rom Collection by ID
Args:
id: the ID of the Rom Collection to be found (as a str)
Returns:
The Rom Collection with the matching ID, or None if not found
"""
try:
return self.romCollections.get(id)
except KeyError as e:
log.warn("Unable to find rom collection with ID {0}".format(id))
return None
def getRomCollectionById(self, id):
"""
Find the matching Rom Collection by ID
Args:
id: the ID of the Rom Collection to be found (as a str)
Returns:
The Rom Collection with the matching ID, or None if not found
"""
try:
return self.romCollections.get(id)
except KeyError as e:
log.warn("Unable to find rom collection with ID {0}".format(id))
return None
def __executeCommand(self, cmd):
# change working directory
path = os.path.dirname(self.romCollection.emulatorCmd)
if os.path.isdir(path):
try:
os.chdir(path)
except OSError:
log.warn("Unable to chdir to {0}".format(path))
if self.romCollection.usePopen:
import subprocess
process = subprocess.Popen(cmd.encode(self.__getEncoding()),
shell=True)
process.wait()
else:
os.system(cmd.encode(self.__getEncoding()))
def getBestResults(self, results, gamenameFromFile):
"""
Compare a game name against each item in a result set to work out which is the likely match
Args:
results: A list of dicts with the SearchKey key being the game title in the result set
gamenameFromFile: The title of the game we are trying to match
Returns:
Either None if no match was found, or the title of the matching game (SearchKey key in the dict)
"""
log.info("getBestResults")
if results is None or len(results) == 0:
log.info("No results found with current scraper")
return None
log.info("Searching for game: " + gamenameFromFile)
log.info("%s results found. Try to find best match." %
str(len(results)))
result = self.matchGamename(results, gamenameFromFile)
if result:
# get name of found result
foundgame = self.resolveParseResult(result, 'SearchKey')
log.info("Using result %s" % foundgame)
return result
# stop searching in accurate mode
if self.update_option == util.SCRAPING_OPTION_AUTO_ACCURATE:
log.warn(
"No game found with scraping option 'Accurate'. Game will be skipped"
)
return None
# Ask for correct result in Interactive mode
if self.update_option == util.SCRAPING_OPTION_INTERACTIVE:
res = self.ask_user_for_result(gamenameFromFile, results)
if res == 0: # Skip Game
log.info("No result chosen by user")
return None
else:
selectedGame = self.resolveParseResult(results[res - 1],
'Game')
log.info("Result chosen by user: " + str(selectedGame))
return results[res - 1]
def get_platform_for_scraper(self, platformname):
"""Get the platform identifier used on the corresponding website.
Args:
platformname: The RCB platform name
Returns:
String that is the identifier for the platform on the corresponding website.
"""
try:
ix = self.pmaps.index(self._name)
except ValueError as e:
# Did not find a mapping
log.warn("Did not find a platform mapping for {0}".format(self._name))
ix = 0
return self.consoleDict[platformname][ix]
def __executeCommand(self, romCollection, cmd):
log.info('__executeCommand')
# change working directory
path = os.path.dirname(romCollection.emulatorCmd)
if os.path.isdir(path):
try:
os.chdir(path)
except OSError:
log.warn("Unable to chdir to {0}".format(path))
if romCollection.usePopen:
log.info('execute command with popen')
import subprocess
process = subprocess.Popen(cmd, shell=True)
process.wait()
else:
log.info('execute command with os.system')
os.system(cmd)
def resolveParseResult(self, result, itemName):
resultValue = u''
try:
resultValue = result[itemName][0]
if (isinstance(resultValue, str)):
resultValue = resultValue.strip()
resultValue = util.convertToUnicodeString(resultValue)
except Exception as exc:
log.warn(u"Error while resolving item: %s: %s" % (itemName, exc))
try:
log.debug(u"Result %s = %s" % (itemName, resultValue))
except:
pass
return resultValue
def getBestResults(self, results, gamenameFromFile):
"""
Compare a game name against each item in a result set to work out which is the likely match
Args:
results: A list of dicts with the SearchKey key being the game title in the result set
gamenameFromFile: The title of the game we are trying to match
Returns:
Either None if no match was found, or the title of the matching game (SearchKey key in the dict)
"""
log.info("getBestResults")
if results is None or len(results) == 0:
log.info("No results found with current scraper")
return None
log.info("Searching for game: " + gamenameFromFile)
log.info("%s results found. Try to find best match." % str(len(results)))
result = self.matchGamename(results, gamenameFromFile, False)
if result:
# get name of found result
foundgame = self.resolveParseResult(result, 'SearchKey')
log.info("Using result %s" % foundgame)
return result
# stop searching in accurate mode
if self.update_option == util.SCRAPING_OPTION_AUTO_ACCURATE:
log.warn("No game found with scraping option 'Accurate'. Game will be skipped")
return None
# Ask for correct result in Interactive mode
if self.update_option == util.SCRAPING_OPTION_INTERACTIVE:
res = self.ask_user_for_result(gamenameFromFile, results)
if res == 0: # Skip Game
log.info("No result chosen by user")
return None
else:
selectedGame = self.resolveParseResult(results[res - 1], 'Game')
log.info("Result chosen by user: " + str(selectedGame))
return results[res - 1]
def matchGamename(self, results, gamenameFromFile, checkSubtitle):
for idx, result in enumerate(results):
try:
# Check if the result has the correct platform (if needed)
found_platform = self.resolveParseResult(result, 'PlatformSearchKey')
if found_platform != '' and self.expected_platform != found_platform:
log.info("Platform mismatch. %s != %s. Result will be skipped." % (
self.expected_platform, found_platform))
continue
searchKey = self.resolveParseResult(result, 'SearchKey')
# keep it for later reference
searchkey_orig = searchKey
gamename_orig = gamenameFromFile
# if no searchkey is specified first result is valid (1 file per game scenario)
if searchkey_orig == '':
log.info("No searchKey found. Using first result")
return result
log.info("Comparing %s with %s" % (gamename_orig, searchkey_orig))
if gamename_orig == searchkey_orig:
# perfect match
return result
# normalize name and searchkey before comparison
gnu = GameNameUtil()
gamename_normalized = gnu.normalize_name(gamename_orig)
searchkey_normalized = gnu.normalize_name(searchkey_orig)
log.info("Try normalized names. Comparing %s with %s" % (gamename_normalized, searchkey_normalized))
if gamename_normalized == searchkey_normalized:
# perfect match
return result
#strip additional info from gamename
gamename_stripped = gnu.strip_addinfo_from_name(gamename_orig)
gamename_stripped = gnu.normalize_name(gamename_stripped)
log.info("Try with stripped additional info. Comparing %s with %s" % (gamename_stripped, searchkey_normalized))
if gamename_stripped == searchkey_normalized:
# perfect match
return result
except Exception, (exc):
log.warn("An error occured while matching the best result: " + str(exc))
def get_platform_for_scraper(self, platformname):
"""Get the platform identifier used on the corresponding website.
Args:
platformname: The RCB platform name
Returns:
String that is the identifier for the platform on the corresponding website.
"""
try:
ix = self.pmaps.index(self._name)
except ValueError as e:
# Did not find a mapping
log.warn("Did not find a platform mapping for {0}".format(
self._name))
ix = 0
return self.consoleDict[platformname][ix]
def _parse_search_results(self, response):
results = []
""" response is expected to be a JSON object """
log.debug("Parsing response for search results: {0}".format(response))
if len(response["games"]) == 0:
log.warn("No results found")
return results
for result in response['games']:
results.append({'id': result['game_id'],
'title': result['title'],
'releaseDate': "", # MobyGames search doesn't return a year in brief mode
'SearchKey': [result['title']]})
log.debug("Found {0} results using requests JSON parser: {1}".format(len(results), results))
return results
def getFilesByGameNameIgnoreCase(self, pathname):
files = []
dirname = os.path.dirname(pathname)
basename = os.path.basename(pathname)
# search all Files that start with the first character of game name
newpath = util.joinPath(dirname, basename[0].upper() + '*')
filesUpper = glob.glob(newpath)
newpath = util.joinPath(dirname, basename[0].lower() + '*')
filesLower = glob.glob(newpath)
allFiles = filesUpper + filesLower
for f in allFiles:
if pathname.lower() == f.lower():
log.warn("Found path '%s' by search with ignore case." % pathname)
files.append(f)
return files
def getFilesByGameNameIgnoreCase(self, pathname):
files = []
dirname = os.path.dirname(pathname)
basename = os.path.basename(pathname)
# search all Files that start with the first character of game name
newpath = util.joinPath(dirname, basename[0].upper() + '*')
filesUpper = glob.glob(newpath)
newpath = util.joinPath(dirname, basename[0].lower() + '*')
filesLower = glob.glob(newpath)
allFiles = filesUpper + filesLower
for file in allFiles:
if pathname.lower() == file.lower():
log.warn("Found path '%s' by search with ignore case." % pathname)
files.append(file)
return files
def insertGameFromDesc(self, gamedescription, gamename, romCollection, filenamelist, foldername, isUpdate, gameId):
log.info("insertGameFromDesc")
if gamedescription is not None:
game = self.resolveParseResult(gamedescription, 'Game')
else:
game = ''
# if no game name has been scraped we expect that no results have been found
if game == '':
self.missingDescFile.add_entry(gamename)
if __addon__.getSetting(util.SETTING_RCB_IGNOREGAMEWITHOUTDESC).upper() == 'TRUE':
log.warn("No description found for game '%s'. Game will not be imported." % gamename)
return None
gamedescription = {}
gameId = self.insertData(gamedescription, gamename, romCollection, filenamelist, foldername, isUpdate, gameId)
return gameId
def addNewElements(self, results, newResults):
""" Add fields from the results to the existing set of results, adding if new, replacing if empty. This allows
us to add fields from subsequent site scrapes that were missing or not available in previous sites.
Args:
results: Existing results dict from previous scrapes
newResults: Result dict from most recent scrape
Returns:
Updated dict of result fields
"""
try:
log.debug("Before merging results: %s vs %s" % (results.items(), newResults.items()))
# Retain any existing key values that aren't an empty list, overwrite all others
z = dict(newResults.items() + dict((k, v) for k, v in results.iteritems() if len(v) > 0).items())
log.debug("After merging results: %s" % z.items())
return z
except Exception as e:
# Return original results without doing anything
log.warn("Error when merging results: %s" % e)
return results
def insertForeignKeyItemList(self, result, itemName, gdbObject):
idList = []
try:
itemList = result[itemName]
log.info("Result %s = %s" % (itemName, itemList))
except KeyError:
log.warn("Error while resolving item: %s" % itemName)
return idList
for item in itemList:
itemRow = gdbObject.getOneByName(item)
if itemRow is None:
log.info("%s does not exist in database. Insert: %s" % (itemName, item.encode('utf-8')))
gdbObject.insert((item,))
idList.append(self.gdb.cursor.lastrowid)
else:
idList.append(itemRow[0])
return idList
def addNewElements(self, results, newResults):
""" Add fields from the results to the existing set of results, adding if new, replacing if empty. This allows
us to add fields from subsequent site scrapes that were missing or not available in previous sites.
Args:
results: Existing results dict from previous scrapes
newResults: Result dict from most recent scrape
Returns:
Updated dict of result fields
"""
try:
log.debug("Before merging results: %s vs %s" % (list(results.items()), list(newResults.items())))
# Retain any existing key values that aren't an empty list, overwrite all others
z = dict(list(newResults.items()) + list(dict((k, v) for k, v in list(results.items()) if len(v) > 0).items()))
log.debug("After merging results: %s" % list(z.items()))
return z
except Exception as e:
# Return original results without doing anything
log.warn("Error when merging results: %s" % e)
return results
def get_platform_for_scraper(self, platformname):
"""Get the platform identifier used on the corresponding website.
Args:
platformname: The RCB platform name
Returns:
String that is the identifier for the platform on the corresponding website.
"""
try:
#HACK: use same platform mapping for thegamesdb and legacy.thegamesdb
#remove, when thegamesdbs update process is finished
ix = self.pmaps.index(self._name)
except ValueError:
# Did not find a mapping
log.warn("Did not find a platform mapping for {0}".format(
self._name))
ix = 0
return self.consoleDict[platformname][ix]
def resolveParseResult(self, result, itemName):
resultValue = ""
try:
resultValue = result[itemName][0]
resultValue = util.html_unescape(resultValue)
resultValue = resultValue.strip()
# unescape ugly html encoding from websites
resultValue = HTMLParser.HTMLParser().unescape(resultValue)
except Exception as e:
# log.warn("Error while resolving item: " + itemName + " : " + str(exc))
log.warn("Error while resolving item: {0} : {1} {2}".format(itemName, type(e), str(e)))
try:
log.debug("Result " + itemName + " = " + resultValue)
except:
pass
return resultValue
def _parse_covers_result(self, response):
result = {}
covergroups = response['cover_groups']
if len(covergroups) == 0:
log.warn("No covers found in mobygames response")
return result
#HACK: always use the first covergroup. We could add an option to search for specific region covers
covergroup = covergroups[0]
for cover in covergroup['covers']:
if cover['scan_of'] == 'Front Cover':
result['Filetypeboxfront'] = [cover['image']]
elif cover['scan_of'] == 'Back Cover':
result['Filetypeboxback'] = [cover['image']]
elif cover['scan_of'] == 'Media':
result['Filetypecartridge'] = [cover['image']]
return result
def insertForeignKeyItemList(self, result, itemName, gdbObject):
idList = []
try:
itemList = result[itemName]
log.info("Result %s = %s" % (itemName, itemList))
except KeyError:
log.warn("Error while resolving item: %s" % itemName)
return idList
for item in itemList:
itemRow = gdbObject.getOneByName(item)
if itemRow is None:
log.info("%s does not exist in database. Insert: %s" % (itemName, item.encode('utf-8')))
gdbObject.insert((item,))
idList.append(self.gdb.cursor.lastrowid)
else:
idList.append(itemRow[0])
return idList
def _parse_game_result(self, response):
""" response is expected to be a JSON object """
result = {}
# Standard fields
for k, v in self._game_mapping.items():
try:
# HACK - for compatibility we need to put each result in an array
# result[k] = response[v]
result[k] = [response[v]]
except KeyError as k:
log.warn("Unable to find key: {0}".format(k))
except Exception as e:
log.warn("Unable to extract data from key {0}".format(e))
# Custom fields (i.e. ones that require special handling)
result['Genre'] = self._parse_genres(response['genres'])
# FIXME TODO Publisher is in releases.companies.@role="Published by"?
return result
def _parse_covers_result(self, response):
result = {}
covergroups = response['cover_groups']
if len(covergroups) == 0:
log.warn("No covers found in mobygames response")
return result
#HACK: always use the first covergroup. We could add an option to search for specific region covers
covergroup = covergroups[0]
for cover in covergroup['covers']:
if cover['scan_of'] == 'Front Cover':
result['Filetypeboxfront'] = [cover['image']]
elif cover['scan_of'] == 'Back Cover':
result['Filetypeboxback'] = [cover['image']]
elif cover['scan_of'] == 'Media':
result['Filetypecartridge'] = [cover['image']]
return result
def _parse_game_result(self, response):
""" response is expected to be a JSON object """
result = {}
# Standard fields
for k, v in self._game_mapping.items():
try:
# HACK - for compatibility we need to put each result in an array
# result[k] = response[v]
result[k] = [response[v]]
except KeyError:
log.warn("Unable to find key: {0}".format(k))
except Exception as e:
log.warn("Unable to extract data from key {0}".format(e))
# Custom fields (i.e. ones that require special handling)
result['Genre'] = self._parse_genres(response['genres'])
# FIXME TODO Publisher is in releases.companies.@role="Published by"?
return result
def _parse_search_results(self, response):
""" response is expected to be a JSON object """
log.debug("Parsing response for search results: {0}".format(response))
results = []
if response['number_of_total_results'] == 0:
log.warn("No results found")
return results
for result in response['results']:
try:
year = self._parse_date(result['release_date'])
results.append({
'id': result['guid'],
'title': result['name'],
'releaseDate': year,
'SearchKey': [result['name']]
})
except KeyError:
log.warn("Unable to find expected field in response")
except Exception as e:
log.warn("Error parsing field: {0}".format(e))
log.debug("Found {0} results using requests JSON parser: {1}".format(
len(results), results))
return results
def __copyLauncherScriptsToUserdata(self):
log.info('__copyLauncherScriptsToUserdata')
oldBasePath = os.path.join(util.getAddonInstallPath(), 'resources', 'scriptfiles')
newBasePath = os.path.join(util.getAddonDataPath(), 'scriptfiles')
files = []
# Copy applaunch shell script/batch file
if self.env == 'win32':
files.append('applaunch.bat')
else:
files.append('applaunch.sh')
# Copy VBS files
if self.env == 'win32' and __addon__.getSetting(util.SETTING_RCB_USEVBINSOLOMODE).lower() == 'true':
files += ['applaunch-vbs.bat', 'LaunchKodi.vbs', 'Sleep.vbs']
for f in files:
if not xbmcvfs.exists(os.path.join(newBasePath, f)):
log.debug("Copying file {0} from {1} to {2}".format(f, oldBasePath, newBasePath))
if not xbmcvfs.copy(os.path.join(oldBasePath, f), os.path.join(newBasePath, f)):
log.warn("Error copying file")
def update(self, columns, argsOrig, obj_id, updateWithNullValues):
if len(columns) != len(argsOrig):
log.warn("len columns != len args in gdb.update()")
return
args = []
updateString = "Update %s SET " % self.tableName
for i in range(0, len(columns)):
#don't update with empty values
if not updateWithNullValues and (argsOrig[i] == '' or argsOrig[i] == None):
continue
if i > 0:
updateString += ", "
args.append(argsOrig[i])
updateString += columns[i] + " = ?"
updateString += " WHERE id = " + str(obj_id)
self.gdb.cursor.execute(updateString, args)
def _parse_search_results(self, response):
""" response is expected to be a JSON object """
log.debug("Parsing response for search results: {0}".format(response))
results = []
if response['number_of_total_results'] == 0:
log.warn("No results found")
return results
for result in response['results']:
try:
year = self._parse_date(result['release_date'])
results.append({'id': result['guid'],
'title': result['name'],
'releaseDate': year,
'SearchKey': [result['name']]})
except KeyError as k:
log.warn("Unable to find expected field in response")
except Exception as e:
log.warn("Error parsing field: {0}".format(e))
log.debug("Found {0} results using requests JSON parser: {1}".format(len(results), results))
return results
log.error("Could not create artwork directory: '%s'" % dirname)
xbmcgui.Dialog().ok(util.localize(32010), util.localize(32011))
del dirname
return False, artworkurls
log.info("File %s does not exist, starting download" % fileName)
# Dialog Status Art Download
# Update progress dialog to state we are downloading art
try:
msg = "%s: %s" % (util.localize(32123), self._guiDict["gameNameKey"])
submsg = "%s - downloading art" % self._guiDict["scraperSiteKey"][thumbKey]
self._gui.writeMsg(self._guiDict["dialogHeaderKey"], msg, submsg, self._guiDict["fileCountKey"])
except KeyError:
log.warn("Unable to retrieve key from GUI dict")
try:
self.download_thumb(thumbUrl, fileName)
except Exception, (exc):
log.error("Could not create file: '%s'. Error message: '%s'" % (fileName, exc))
# xbmcgui.Dialog().ok(util.localize(32012), util.localize(32011))
return False, artworkurls
Logutil.log("Download finished.", util.LOG_LEVEL_INFO)
except Exception, (exc):
log.warn("Error in getThumbFromOnlineSource: %s" % exc)
return True, artworkurls
def getThumbFromOnlineSource(self, gamedescription, fileType, fileName, artworkurls):
log.info("Get thumb from online source")
try:
# maybe we got a thumb url from desc parser
thumbKey = 'Filetype' + fileType
log.info("using key: %s" % thumbKey)
thumbUrl = self.resolveParseResult(gamedescription, thumbKey)
if thumbUrl == '':
return True, artworkurls
artworkurls[fileType] = thumbUrl
log.info("Get thumb from url: %s" % thumbUrl)
rootExtFile = os.path.splitext(fileName)
rootExtUrl = os.path.splitext(thumbUrl)
files = []
if len(rootExtUrl) == 2 and len(rootExtFile) != 0:
fileName = rootExtFile[0] + rootExtUrl[1]
gameName = rootExtFile[0] + ".*"
files = self.getFilesByWildcard(gameName)
del rootExtFile, rootExtUrl
if len(files) > 0:
log.info("File already exists. Won't download again.")
return True, artworkurls
# Create folder if it doesn't already exist
dirname = os.path.join(os.path.dirname(fileName), '') # Add the trailing slash that xbmcvfs.exists expects
log.debug("Checking for artwork directory %s" % dirname)
if KodiVersions.getKodiVersion() >= KodiVersions.KRYPTON:
exists = xbmcvfs.exists(dirname)
else:
exists = os.path.exists(dirname)
if not exists:
log.info("Artwork directory %s doesn't exist, creating it" % dirname)
success = xbmcvfs.mkdirs(dirname)
log.info("Directory successfully created: %s" %success)
if not success:
#HACK: check if directory was really not created.
directoryExists = xbmcvfs.exists(dirname)
log.info("Directory exists: %s" %directoryExists)
if not directoryExists:
log.error("Could not create artwork directory: '%s'" % dirname)
xbmcgui.Dialog().ok(util.localize(32010), util.localize(32011))
del dirname
return False, artworkurls
log.info("File %s does not exist, starting download" % fileName)
# Dialog Status Art Download
# Update progress dialog to state we are downloading art
try:
msg = "%s: %s" % (util.localize(32123), self._guiDict["gameNameKey"])
submsg = "%s - downloading art" % self._guiDict["scraperSiteKey"][thumbKey]
self._gui.writeMsg(self._guiDict["dialogHeaderKey"], msg, submsg, self._guiDict["fileCountKey"])
except KeyError:
log.warn("Unable to retrieve key from GUI dict")
try:
self.download_thumb(thumbUrl, fileName)
except Exception, (exc):
log.error("Could not create file: '%s'. Error message: '%s'" % (fileName, exc))
# xbmcgui.Dialog().ok(util.localize(32012), util.localize(32011))
return False, artworkurls
Logutil.log("Download finished.", util.LOG_LEVEL_INFO)
elif len(names) > 1:
log.info("The Archive has %d files" % len(names))
chosenROM = xbmcgui.Dialog().select('Choose a ROM', names)
elif len(names) == 1:
log.info("Archive only has one file inside; picking that one")
chosenROM = 0
else:
log.error("Archive had no files inside!")
return []
if chosenROM != -1:
# Extract all files to %TMP%
archives = self.__getArchives(filext, rom, names)
if archives is None:
log.warn("Error handling compressed file")
return []
for archive in archives:
newPath = os.path.join(tempDir, archive[0])
log.info("Putting extracted file in %s" % newPath)
fo = open(str(newPath), 'wb')
fo.write(archive[1])
fo.close()
# Point file name to the chosen file and continue as usual
roms = [os.path.join(tempDir, names[chosenROM])]
return roms
def __replacePlaceholdersInParams(self, emuParams, rom, gameRow):
def updateDB(self, gdb, gui, romCollections, isRescrape):
self.gdb = gdb
self._gui = gui
log.info("Start Update DB")
log.info("Iterating Rom Collections")
rccount = 1
# always do full reimports when in rescrape-mode
enableFullReimport = isRescrape or __addon__.getSetting(util.SETTING_RCB_ENABLEFULLREIMPORT).upper() == 'TRUE'
log.info("enableFullReimport: {0}".format(enableFullReimport))
continueUpdate = True
# Added variable to allow user to continue on errors
ignoreErrors = False
for romCollection in romCollections.values():
# timestamp1 = time.clock()
# check if import was canceled
if not continueUpdate:
log.info("Game import canceled")
break
# prepare Header for ProgressDialog
progDialogRCHeader = util.localize(32122) + " (%i / %i): %s" % (
rccount, len(romCollections), romCollection.name)
rccount += 1
log.info("current Rom Collection: {0}".format(romCollection.name))
# Read settings for current Rom Collection
log.info("ignoreOnScan: {0}".format(romCollection.ignoreOnScan))
if romCollection.ignoreOnScan:
log.info("current Rom Collection will be ignored.")
# self.scrapeResultsFile.write('Rom Collection will be ignored.\n')
continue
log.info("update is allowed for current rom collection: {0}".format(romCollection.allowUpdate))
log.info("max folder depth: {0}".format(romCollection.maxFolderDepth))
files = self.getRomFilesByRomCollection(romCollection, enableFullReimport)
if len(files) == 0:
log.info(u"No files found for rom collection {0}, skipping".format(romCollection.name))
continue
log.info(u"Found {0} game files for rom collection {1}".format(len(files), romCollection.name))
# itemCount is used for percentage in ProgressDialogGUI
self._gui.itemCount = len(files) + 1
successfulFiles = 0
lastgamename = ''
lastGameId = None
for fileidx, filename in enumerate(files):
try:
#Give kodi a chance to interrupt the process
#HACK: we should use monitor.abortRequested() or monitor.waitForAbort()
#but for some reason only xbmc.abortRequested returns True
if monitor.abortRequested() or xbmc.abortRequested:
log.info("Kodi requests abort. Cancel Update.")
break
log.info("Scraping for %s" % filename)
gamenameFromFile = romCollection.getGamenameFromFilename(filename)
# check if we are handling one of the additional disks of a multi rom game
isMultiRomGame = (gamenameFromFile == lastgamename)
lastgamename = gamenameFromFile
if isMultiRomGame:
# Add this entry as a file under the game ID and move on
log.info("Detected %s as a multirom game (previous game was %s)" % (filename, lastgamename))
if lastGameId is None:
log.error("Game detected as multi rom game, but lastGameId is None.")
continue
fileType = FileType()
fileType.id, fileType.name, fileType.parent = 0, "rcb_rom", "game"
self.insertFile(filename, lastGameId, fileType, None, None, None)
self.gdb.commit()
del fileType
continue
log.info("Start scraping info for game: %s" % gamenameFromFile)
continueUpdate = self._gui.writeMsg(progDialogRCHeader,
util.localize(32123) + ": " + gamenameFromFile, "", fileidx + 1)
if not continueUpdate:
log.info("Game import canceled by user")
break
# check if this file already exists in DB
continueUpdate, isUpdate, gameId = self.checkRomfileAlreadyExists(filename, enableFullReimport)
if not continueUpdate:
continue
results = {}
foldername = self.getFoldernameFromRomFilename(filename)
artScrapers = {}
results, artScrapers = self.useSingleScrapers(romCollection, filename, gamenameFromFile,
progDialogRCHeader, fileidx + 1)
if len(results) == 0:
# lastgamename = ""
results = None
# Variables to process Art Download Info
self._guiDict.update({'dialogHeaderKey': progDialogRCHeader, 'gameNameKey': gamenameFromFile,
'scraperSiteKey': artScrapers, 'fileCountKey': (fileidx + 1)})
del artScrapers
#Give kodi a chance to interrupt the process
#HACK: we should use monitor.abortRequested() or monitor.waitForAbort()
#but for some reason only xbmc.abortRequested returns True
if monitor.abortRequested() or xbmc.abortRequested:
log.info("Kodi requests abort. Cancel Update.")
break
# Add 'gui' and 'dialogDict' parameters to function
lastGameId = self.insertGameFromDesc(results, gamenameFromFile, romCollection, [filename],
foldername, isUpdate, gameId)
del results, foldername
if lastGameId is not None:
log.info("Successfully added %s" % gamenameFromFile)
successfulFiles += 1
# Check if all first 10 games have errors - Modified to allow user to continue on errors
if fileidx > 9 and successfulFiles == 0 and not ignoreErrors:
options = [util.localize(32124), util.localize(32125), util.localize(32126)]
answer = xbmcgui.Dialog().select(util.localize(32127), options)
if answer == 1:
# Continue and ignore errors
ignoreErrors = True
elif answer == 2:
# Cancel
xbmcgui.Dialog().ok(util.SCRIPTNAME, util.localize(32128), util.localize(32129))
continueUpdate = False
break
except ScraperExceededAPIQuoteException as ke:
xbmcgui.Dialog().ok(util.localize(32128), "The API key for a scraper was exceeded")
# Abort the scraping entirely
break
except Exception as exc:
log.warn(u"An error occurred while adding game %s: %s" % (gamenameFromFile, exc))
self.missingDescFile.add_entry(gamenameFromFile)
continue
# timestamp2 = time.clock()
# diff = (timestamp2 - timestamp1) * 1000
# print "load %i games in %d ms" % (self.getListSize(), diff)
self._gui.writeMsg("Done.", "", "", self._gui.itemCount)
log.info("Update finished")
return True, ''
def insertData(self, gamedescription, gamenameFromFile, romCollection, romFiles, foldername, isUpdate, gameId):
log.info("Insert data")
publisher = self.resolveParseResult(gamedescription, 'Publisher')
developer = self.resolveParseResult(gamedescription, 'Developer')
year = self.resolveParseResult(gamedescription, 'ReleaseYear')
yearId = self.insertForeignKeyItem(gamedescription, 'ReleaseYear', Year(self.gdb))
genreIds = self.insertForeignKeyItemList(gamedescription, 'Genre', Genre(self.gdb))
reviewerId = self.insertForeignKeyItem(gamedescription, 'Reviewer', Reviewer(self.gdb))
publisherId = self.insertForeignKeyItem(gamedescription, 'Publisher', Publisher(self.gdb))
developerId = self.insertForeignKeyItem(gamedescription, 'Developer', Developer(self.gdb))
region = self.resolveParseResult(gamedescription, 'Region')
media = self.resolveParseResult(gamedescription, 'Media')
controller = self.resolveParseResult(gamedescription, 'Controller')
players = self.resolveParseResult(gamedescription, 'Players')
rating = self.resolveParseResult(gamedescription, 'Rating')
votes = self.resolveParseResult(gamedescription, 'Votes')
url = self.resolveParseResult(gamedescription, 'URL')
perspective = self.resolveParseResult(gamedescription, 'Perspective')
originalTitle = self.resolveParseResult(gamedescription, 'OriginalTitle')
alternateTitle = self.resolveParseResult(gamedescription, 'AlternateTitle')
translatedBy = self.resolveParseResult(gamedescription, 'TranslatedBy')
version = self.resolveParseResult(gamedescription, 'Version')
plot = self.resolveParseResult(gamedescription, 'Description')
isFavorite = self.resolveParseResult(gamedescription, 'IsFavorite')
if isFavorite == '':
isFavorite = '0'
launchCount = self.resolveParseResult(gamedescription, 'LaunchCount')
if launchCount == '':
launchCount = '0'
if gamedescription is not None:
gamename = self.resolveParseResult(gamedescription, 'Game')
if gamename != gamenameFromFile:
self.possibleMismatchFile.add_entry(gamename, gamenameFromFile)
if gamename == "":
gamename = gamenameFromFile
else:
gamename = gamenameFromFile
artWorkFound, artworkfiles, artworkurls = self.getArtworkForGame(romCollection, gamename, gamenameFromFile,
gamedescription, foldername, publisher,
developer)
# Create Nfo file with game properties
try:
genreList = gamedescription.get('Genre', [])
writer = NfoWriter()
writer.createNfoFromDesc(gamename, plot, romCollection.name, publisher, developer, year,
players, rating, votes, url, region, media, perspective, controller,
originalTitle, alternateTitle, version, genreList, isFavorite, launchCount,
romFiles[0], gamenameFromFile, artworkfiles, artworkurls)
except Exception as e:
log.warn(u"Unable to write NFO file for game %s: %s" % (gamename, e))
del publisher, developer, year
gameId = self.insertGame(gamename, plot, romCollection.id, publisherId, developerId, reviewerId, yearId,
players, rating, votes, url, region, media, perspective, controller, originalTitle,
alternateTitle, translatedBy, version, isFavorite, launchCount, isUpdate, gameId,
romCollection.allowUpdate, )
del plot, players, rating, votes, url, region, media, perspective, controller, originalTitle, alternateTitle, translatedBy, version
if gameId is None:
return None
self.add_genres_to_db(genreIds, gameId)
self.add_romfiles_to_db(romFiles, gameId)
for fileType, fileNames in artworkfiles.iteritems():
for filename in fileNames:
log.info("Importing artwork file %s = %s" % (fileType.type, filename))
self.insertFile(filename, gameId, fileType, romCollection.id, publisherId, developerId)
self.gdb.commit()
return gameId
def useSingleScrapers(self, romCollection, romFile, gamenameFromFile, progDialogRCHeader, fileCount):
"""Scrape site for game metadata
Args:
romCollection:
gamenameFromFile:
progDialogRCHeader:
fileCount:
Returns:
dict for the game result:
{'SearchKey': ['Chrono Trigger'],
'Publisher': ['Squaresoft'],
'Description': ["The millennium. A portal is opened. The chain of time is broken...],
'Players': ['1'],
'Platform': ['Super Nintendo (SNES)'],
'Game': ['Chrono Trigger'],
'Filetypeboxfront': ['http://thegamesdb.net/banners/boxart/original/front/1255-1.jpg'],
'Filetypeboxback': ['http://thegamesdb.net/banners/boxart/original/back/1255-1.jpg'],
'Filetypescreenshot': ['http://thegamesdb.net/banners/screenshots/1255-1.jpg', 'http://thegamesdb.net/banners/screenshots/1255-2.jpg', 'http://thegamesdb.net/banners/screenshots/1255-3.jpg', 'http://thegamesdb.net/banners/screenshots/1255-4.jpg', 'http://thegamesdb.net/banners/screenshots/1255-5.jpg'],
'Filetypefanart': ['http://thegamesdb.net/banners/fanart/original/1255-1.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-10.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-11.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-2.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-3.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-4.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-5.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-6.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-7.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-8.jpg', 'http://thegamesdb.net/banners/fanart/original/1255-9.jpg'],
'Genre': ['Role-Playing'],
'Developer': ['Squaresoft']}
dict for artwork urls:
{'Filetypefanart': 'thegamesdb.net', 'Filetypeboxback': 'thegamesdb.net', 'Filetypescreenshot': 'thegamesdb.net', 'Filetypeboxfront': 'thegamesdb.net'}
Note - this only contains entries for artwork that was found (i.e. is not empty list)
"""
gameresult = {}
artScrapers = {}
scraperSite = romCollection.scraperSites[0]
try:
#first check if a local nfo file is available
nfoscraper = NFO_Scraper()
nfofile = nfoscraper.get_nfo_path(gamenameFromFile, romCollection.name, romFile)
if xbmcvfs.exists(nfofile) and __addon__.getSetting(util.SETTING_RCB_PREFERLOCALNFO).upper() == 'TRUE':
log.info("Found local nfo file. Using this to scrape info.")
newscraper = nfoscraper
else:
newscraper = AbstractScraper().get_scraper_by_name(scraperSite.name)
#set path to desc file (only useful for offline scrapers)
newscraper.path = scraperSite.path
results = newscraper.search(gamenameFromFile, romCollection.name)
log.debug(u"Searching for %s - found %s results: %s" % (gamenameFromFile, len(results), results))
except ScraperExceededAPIQuoteException as ke:
# API key is invalid - we need to stop scraping
log.error("Scraper exceeded API key, stopping scraping")
raise
except Exception as e:
log.error("Error searching for %s using scraper %s - %s %s" % (
gamenameFromFile, scraperSite.name, type(e), e))
return gameresult, artScrapers
if results == []:
log.warn("No search results found for %s using scraper %s" % (gamenameFromFile, scraperSite.name))
return gameresult, artScrapers
matched = Matcher().getBestResults(results, gamenameFromFile)
if matched is None:
log.error("No matches found for %s, skipping" % gamenameFromFile)
return gameresult, artScrapers
log.debug("After matching: %s" % matched)
try:
retrievedresult = newscraper.retrieve(matched['id'], romCollection.name)
log.debug(u"Retrieving %s - found %s" % (matched['id'], retrievedresult))
except Exception as e:
# FIXME TODO Catch exceptions specifically
log.error("Error retrieving %s - %s %s" % (matched['id'], type(e), e))
return gameresult, artScrapers
# Update the gameresult with any new fields
gameresult = self.addNewElements(gameresult, retrievedresult)
self._gui.writeMsg(progDialogRCHeader, util.localize(32123) + ": " + gamenameFromFile,
scraperSite.name + " - " + util.localize(32131), fileCount)
# Find Filetypes and Scrapers for Art Download
# FIXME TODO The following is kept to keep artwork downloading working as it currently is. We already have
# the URLs and so could handle/download here, rather than deferring
if len(gameresult) > 0:
for path in romCollection.mediaPaths:
thumbKey = 'Filetype' + path.fileType.name
if len(self.resolveParseResult(gameresult, thumbKey)) > 0:
if (thumbKey in artScrapers) == 0:
artScrapers[thumbKey] = scraperSite.name
log.debug(u"After scraping, result = %s, artscrapers = %s" % (gameresult, artScrapers))
return gameresult, artScrapers
