def buildFileDict(self, gui, progDialogRCHeader, files, romCollection, firstScraper):
fileCount = 1
lastgamename = ""
crcOfFirstGame = {}
fileDict = {}
for filename in files:
try:
gui.writeMsg(progDialogRCHeader, util.localize(40030), "", fileCount)
fileCount = fileCount +1
gamename = helper.getGamenameFromFilename(filename, romCollection)
#check if we are handling one of the additional disks of a multi rom game
isMultiRomGame = self.checkRomfileIsMultirom(gamename, lastgamename)
#lastgamename may be overwritten by parsed gamename
lastgamename = gamename
gamename = gamename.strip()
gamename = gamename.lower()
#Logutil.log('gamename in fileDict: ' +str(gamename), util.LOG_LEVEL_INFO)
#build dictionaries (key=gamename, filecrc or foldername; value=filenames) for later game search
if(firstScraper.useFoldernameAsCRC):
Logutil.log('useFoldernameAsCRC = True', util.LOG_LEVEL_INFO)
foldername = self.getFoldernameFromRomFilename(filename)
foldername = foldername.strip()
foldername = foldername.lower()
fileDict = self.buildFilenameDict(fileDict, isMultiRomGame, filename, foldername)
elif(firstScraper.useFilenameAsCRC):
Logutil.log('useFilenameAsCRC = True', util.LOG_LEVEL_INFO)
fileDict = self.buildFilenameDict(fileDict, isMultiRomGame, filename, gamename)
elif(firstScraper.searchGameByCRC):
Logutil.log('searchGameByCRC = True', util.LOG_LEVEL_INFO)
filecrc = self.getFileCRC(filename)
#use crc of first rom if it is a multirom game
if(not isMultiRomGame):
crcOfFirstGame[gamename] = filecrc
Logutil.log('Adding crc to crcOfFirstGame-dict: %s: %s' %(gamename, filecrc), util.LOG_LEVEL_INFO)
else:
filecrc = crcOfFirstGame[gamename]
Logutil.log('Read crc from crcOfFirstGame-dict: %s: %s' %(gamename, filecrc), util.LOG_LEVEL_INFO)
fileDict = self.buildFilenameDict(fileDict, isMultiRomGame, filename, filecrc)
else:
fileDict = self.buildFilenameDict(fileDict, isMultiRomGame, filename, gamename)
except Exception, (exc):
Logutil.log("an error occured while building file list", util.LOG_LEVEL_WARNING)
Logutil.log("Error: " +str(exc), util.LOG_LEVEL_WARNING)
continue
def updateDB(self, gdb, gui, updateOption, romCollections, settings, isRescrape):
self.gdb = gdb
self.Settings = settings
#self.scrapeResultsFile = self.openFile(os.path.join(util.getAddonDataPath(), 'scrapeResults.txt'))
self.missingDescFile = self.openFile(os.path.join(util.getAddonDataPath(), 'scrapeResult_missingDesc.txt'))
self.missingArtworkFile = self.openFile(os.path.join(util.getAddonDataPath(), 'scrapeResult_missingArtwork.txt'))
self.possibleMismatchFile = self.openFile(os.path.join(util.getAddonDataPath(), 'scrapeResult_possibleMismatches.txt'))
Logutil.log("Start Update DB", util.LOG_LEVEL_INFO)
Logutil.log("Iterating Rom Collections", util.LOG_LEVEL_INFO)
rccount = 1
#get fuzzyFactor before scraping
matchingRatioIndex = self.Settings.getSetting(util.SETTING_RCB_FUZZYFACTOR)
if (matchingRatioIndex == ''):
matchingRatioIndex = 2
matchingRatioIndex = self.Settings.getSetting(util.SETTING_RCB_FUZZYFACTOR)
Logutil.log("matchingRatioIndex: " +str(matchingRatioIndex), util.LOG_LEVEL_INFO)
fuzzyFactor = util.FUZZY_FACTOR_ENUM[int(matchingRatioIndex)]
Logutil.log("fuzzyFactor: " +str(fuzzyFactor), util.LOG_LEVEL_INFO)
#always do full reimports when in rescrape-mode
enableFullReimport = isRescrape or self.Settings.getSetting(util.SETTING_RCB_ENABLEFULLREIMPORT).upper() == 'TRUE'
Logutil.log("enableFullReimport: " +str(enableFullReimport), util.LOG_LEVEL_INFO)
continueUpdate = True
#Added variable to allow user to continue on errors
ignoreErrors = False
for romCollection in romCollections.values():
#timestamp1 = time.clock()
#check if import was canceled
if(not continueUpdate):
Logutil.log('Game import canceled', util.LOG_LEVEL_INFO)
break
#prepare Header for ProgressDialog
progDialogRCHeader = util.localize(40022) +" (%i / %i): %s" %(rccount, len(romCollections), romCollection.name)
rccount = rccount + 1
Logutil.log("current Rom Collection: " +romCollection.name, util.LOG_LEVEL_INFO)
#self.scrapeResultsFile.write('~~~~~~~~~~~~~~~~~~~~~~~~\n' +romCollection.name +'\n' +'~~~~~~~~~~~~~~~~~~~~~~~~\n')
self.missingDescFile.write('~~~~~~~~~~~~~~~~~~~~~~~~\n' +romCollection.name +'\n' +'~~~~~~~~~~~~~~~~~~~~~~~~\n')
self.missingArtworkFile.write('~~~~~~~~~~~~~~~~~~~~~~~~\n' +romCollection.name +'\n' +'~~~~~~~~~~~~~~~~~~~~~~~~\n')
self.possibleMismatchFile.write('~~~~~~~~~~~~~~~~~~~~~~~~\n' +romCollection.name +'\n' +'~~~~~~~~~~~~~~~~~~~~~~~~\n')
self.possibleMismatchFile.write('gamename, filename\n')
#Read settings for current Rom Collection
Logutil.log("ignoreOnScan: " +str(romCollection.ignoreOnScan), util.LOG_LEVEL_INFO)
if(romCollection.ignoreOnScan):
Logutil.log("current Rom Collection will be ignored.", util.LOG_LEVEL_INFO)
#self.scrapeResultsFile.write('Rom Collection will be ignored.\n')
continue
Logutil.log("update is allowed for current rom collection: " +str(romCollection.allowUpdate), util.LOG_LEVEL_INFO)
Logutil.log("max folder depth: " +str(romCollection.maxFolderDepth), util.LOG_LEVEL_INFO)
firstScraper = romCollection.scraperSites[0]
#check if we are in local artwork mode
if(len(romCollection.scraperSites) == 1 and firstScraper.name == util.localize(40053)):
Logutil.log("Forcing enableFullReimport because we are in local artwork mode", util.LOG_LEVEL_INFO)
enableFullReimport = True
files = self.getRomFilesByRomCollection(romCollection, enableFullReimport)
#itemCount is used for percentage in ProgressDialogGUI
gui.itemCount = len(files) +1
#check if first scraper is a multigame scraper
if(not firstScraper.descFilePerGame):
#build file hash tables (key = gamename or crc, value = romfiles)
Logutil.log("Start building file dict", util.LOG_LEVEL_INFO)
fileDict = self.buildFileDict(gui, progDialogRCHeader, files, romCollection, firstScraper)
try:
fileCount = 0
gamenameFromDesc = ''
#TODO move to to check preconditions
#first scraper must be the one for multiple games
if(len(firstScraper.scrapers) == 0):
Logutil.log('Configuration error: Configured scraper site does not contain any scrapers', util.LOG_LEVEL_ERROR)
continue
scraper = firstScraper.scrapers[0]
Logutil.log("start parsing with multi game scraper: " +str(firstScraper.name), util.LOG_LEVEL_INFO)
Logutil.log("using parser file: " +scraper.parseInstruction, util.LOG_LEVEL_INFO)
Logutil.log("using game description: " +scraper.source, util.LOG_LEVEL_INFO)
parser = DescriptionParserFactory.getParser(str(scraper.parseInstruction))
#parse description
for result in parser.scanDescription(scraper.source, str(scraper.parseInstruction), scraper.encoding):
try:
gamenameFromDesc = result['Game'][0]
#find parsed game in Rom Collection
filenamelist = self.matchDescriptionWithRomfiles(firstScraper, result, fileDict, gamenameFromDesc)
artScrapers = {}
if(filenamelist != None and len(filenamelist) > 0):
gamenameFromFile = helper.getGamenameFromFilename(filenamelist[0], romCollection)
foldername = self.getFoldernameFromRomFilename(filenamelist[0])
fileCount = fileCount +1
continueUpdate = gui.writeMsg(progDialogRCHeader, util.localize(40023) +": " +str(gamenameFromDesc), "", fileCount)
if(not continueUpdate):
Logutil.log('Game import canceled by user', util.LOG_LEVEL_INFO)
break
Logutil.log('Start scraping info for game: ' +str(gamenameFromFile), LOG_LEVEL_INFO)
#check if this file already exists in DB
continueUpdate, isUpdate, gameId = self.checkRomfileAlreadyExists(filenamelist[0], enableFullReimport, False)
if(not continueUpdate):
continue
#use additional scrapers
if(len(romCollection.scraperSites) > 1):
result, artScrapers = self.useSingleScrapers(result, romCollection, 1, gamenameFromFile, foldername, filenamelist[0], fuzzyFactor, updateOption, gui, progDialogRCHeader, fileCount)
else:
Logutil.log("game " +gamenameFromDesc +" was found in parsed results but not in your rom collection.", util.LOG_LEVEL_WARNING)
continue
dialogDict = {'dialogHeaderKey':progDialogRCHeader, 'gameNameKey':gamenameFromFile, 'scraperSiteKey':artScrapers, 'fileCountKey':fileCount}
gameId, continueUpdate = self.insertGameFromDesc(result, gamenameFromFile, romCollection, filenamelist, foldername, isUpdate, gameId, gui, False, dialogDict)
if(not continueUpdate):
break
#remove found files from file list
if(gameId != None):
for filename in filenamelist:
files.remove(filename)
#stop import if no files are left
if(len(files) == 0):
Logutil.log("All games are imported", util.LOG_LEVEL_INFO)
break
except Exception, (exc):
Logutil.log("an error occured while adding game " +gamenameFromDesc, util.LOG_LEVEL_WARNING)
Logutil.log("Error: " +str(exc), util.LOG_LEVEL_WARNING)
continue
#all files still available files-list, are missing entries
for filename in files:
gamenameFromFile = helper.getGamenameFromFilename(filename, romCollection)
try:
self.missingDescFile.write('%s\n' %gamenameFromFile)
except:
self.missingDescFile.write('%s\n' %gamenameFromFile.encode('utf-8'))
except Exception, (exc):
Logutil.log("an error occured while adding game " +gamenameFromDesc, util.LOG_LEVEL_WARNING)
Logutil.log("Error: " +str(exc), util.LOG_LEVEL_WARNING)
try:
self.missingDescFile.write('%s\n' %gamenameFromDesc)
except:
self.missingDescFile.write('%s\n' %gamenameFromDesc.encode('utf-8'))
continue
def exportLibrary(self, gui):
Logutil.log("Begin exportLibrary", util.LOG_LEVEL_INFO)
gdb = gui.gdb
romCollections = gui.config.romCollections
progressDialog = dialogprogress.ProgressDialogGUI()
progressDialog.writeMsg(util.localize(32169), "", "")
continueExport = True
rccount = 1
for romCollection in gui.config.romCollections.values():
progDialogRCHeader = util.localize(32170) +" (%i / %i): %s" %(rccount, len(romCollections), romCollection.name)
rccount = rccount + 1
Logutil.log("export Rom Collection: " +romCollection.name, util.LOG_LEVEL_INFO)
gameCount = 1
#get all games for this Rom Collection
games = Game(gdb).getFilteredGames(romCollection.id, 0, 0, 0, False, '0 = 0')
progressDialog.itemCount = len(games) +1
for gameRow in games:
gamename = self.getGameProperty(gameRow[util.ROW_NAME])
continueExport = progressDialog.writeMsg(progDialogRCHeader, util.localize(32171) +": " +str(gamename), "", gameCount)
if(not continueExport):
Logutil.log('Game export canceled by user', util.LOG_LEVEL_INFO)
break
gameCount = gameCount +1
plot = self.getGameProperty(gameRow[util.GAME_description])
publisher = self.getGamePropertyFromCache(gameRow, gui.publisherDict, util.GAME_publisherId, util.ROW_NAME)
developer = self.getGamePropertyFromCache(gameRow, gui.developerDict, util.GAME_developerId, util.ROW_NAME)
year = self.getGamePropertyFromCache(gameRow, gui.yearDict, util.GAME_yearId, util.ROW_NAME)
genreList = []
try:
cachingOptionStr = self.Settings.getSetting(util.SETTING_RCB_CACHINGOPTION)
if(cachingOptionStr == 'CACHEALL'):
genre = gui.genreDict[gameRow[util.ROW_ID]]
else:
genres = Genre(gdb).getGenresByGameId(gameRow[util.ROW_ID])
if (genres != None):
for i in range(0, len(genres)):
genreRow = genres[i]
genreList.append(genreRow[util.ROW_NAME])
except:
pass
players = self.getGameProperty(gameRow[util.GAME_maxPlayers])
rating = self.getGameProperty(gameRow[util.GAME_rating])
votes = self.getGameProperty(gameRow[util.GAME_numVotes])
url = self.getGameProperty(gameRow[util.GAME_url])
region = self.getGameProperty(gameRow[util.GAME_region])
media = self.getGameProperty(gameRow[util.GAME_media])
perspective = self.getGameProperty(gameRow[util.GAME_perspective])
controller = self.getGameProperty(gameRow[util.GAME_controllerType])
originalTitle = self.getGameProperty(gameRow[util.GAME_originalTitle])
alternateTitle = self.getGameProperty(gameRow[util.GAME_alternateTitle])
version = self.getGameProperty(gameRow[util.GAME_version])
#user settings
isFavorite = self.getGameProperty(gameRow[util.GAME_isFavorite])
launchCount = self.getGameProperty(gameRow[util.GAME_launchCount])
romFiles = File(gdb).getRomsByGameId(gameRow[util.ROW_ID])
romFile = ''
if(romFiles != None and len(romFiles) > 0):
romFile = romFiles[0][0]
gamenameFromFile = helper.getGamenameFromFilename(romFile, romCollection)
artworkfiles = {}
artworkurls = []
self.createNfoFromDesc(gamename, plot, romCollection.name, publisher, developer, year,
players, rating, votes, url, region, media, perspective, controller, originalTitle, alternateTitle, version, genreList, isFavorite, launchCount, romFile, gamenameFromFile, artworkfiles, artworkurls)
progressDialog.writeMsg("", "", "", -1)
del progressDialog
try:
self.missingDescFile.write('%s\n' %gamenameFromDesc)
except:
self.missingDescFile.write('%s\n' %gamenameFromDesc.encode('utf-8'))
continue
else:
fileCount = 0
successfulFiles = 0
lastgamename = ''
lastGameId = None
for filename in files:
try:
gamenameFromFile = ''
gamenameFromFile = helper.getGamenameFromFilename(filename, romCollection)
#check if we are handling one of the additional disks of a multi rom game
isMultiRomGame = self.checkRomfileIsMultirom(gamenameFromFile, lastgamename)
lastgamename = gamenameFromFile
if(isMultiRomGame):
if(lastGameId == None):
Logutil.log('Game detected as multi rom game, but lastGameId is None.', util.LOG_LEVEL_ERROR)
continue
fileType = FileType()
fileType.id = 0
fileType.name = "rcb_rom"
fileType.parent = "game"
self.insertFile(filename, lastGameId, fileType, None, None, None)
continue
def exportLibrary(self, gui):
Logutil.log("Begin exportLibrary", util.LOG_LEVEL_INFO)
gdb = gui.gdb
romCollections = gui.config.romCollections
progressDialog = dialogprogress.ProgressDialogGUI()
progressDialog.writeMsg(util.localize(32169), "", "")
continueExport = True
rccount = 1
for romCollection in gui.config.romCollections.values():
progDialogRCHeader = util.localize(32170) + " (%i / %i): %s" % (
rccount, len(romCollections), romCollection.name)
rccount = rccount + 1
Logutil.log("export Rom Collection: " + romCollection.name,
util.LOG_LEVEL_INFO)
gameCount = 1
#get all games for this Rom Collection
games = Game(gdb).getFilteredGames(romCollection.id, 0, 0, 0,
False, '0 = 0')
progressDialog.itemCount = len(games) + 1
for gameRow in games:
gamename = self.getGameProperty(gameRow[util.ROW_NAME])
continueExport = progressDialog.writeMsg(
progDialogRCHeader,
util.localize(32171) + ": " + str(gamename), "", gameCount)
if (not continueExport):
Logutil.log('Game export canceled by user',
util.LOG_LEVEL_INFO)
break
gameCount = gameCount + 1
plot = self.getGameProperty(gameRow[util.GAME_description])
publisher = self.getGamePropertyFromCache(
gameRow, gui.publisherDict, util.GAME_publisherId,
util.ROW_NAME)
developer = self.getGamePropertyFromCache(
gameRow, gui.developerDict, util.GAME_developerId,
util.ROW_NAME)
year = self.getGamePropertyFromCache(gameRow, gui.yearDict,
util.GAME_yearId,
util.ROW_NAME)
genreList = []
try:
cachingOptionStr = self.Settings.getSetting(
util.SETTING_RCB_CACHINGOPTION)
if (cachingOptionStr == 'CACHEALL'):
genre = gui.genreDict[gameRow[util.ROW_ID]]
else:
genres = Genre(gdb).getGenresByGameId(
gameRow[util.ROW_ID])
if (genres != None):
for i in range(0, len(genres)):
genreRow = genres[i]
genreList.append(genreRow[util.ROW_NAME])
except:
pass
players = self.getGameProperty(gameRow[util.GAME_maxPlayers])
rating = self.getGameProperty(gameRow[util.GAME_rating])
votes = self.getGameProperty(gameRow[util.GAME_numVotes])
url = self.getGameProperty(gameRow[util.GAME_url])
region = self.getGameProperty(gameRow[util.GAME_region])
media = self.getGameProperty(gameRow[util.GAME_media])
perspective = self.getGameProperty(
gameRow[util.GAME_perspective])
controller = self.getGameProperty(
gameRow[util.GAME_controllerType])
originalTitle = self.getGameProperty(
gameRow[util.GAME_originalTitle])
alternateTitle = self.getGameProperty(
gameRow[util.GAME_alternateTitle])
version = self.getGameProperty(gameRow[util.GAME_version])
#user settings
isFavorite = self.getGameProperty(
gameRow[util.GAME_isFavorite])
launchCount = self.getGameProperty(
gameRow[util.GAME_launchCount])
romFiles = File(gdb).getRomsByGameId(gameRow[util.ROW_ID])
romFile = ''
if (romFiles != None and len(romFiles) > 0):
romFile = romFiles[0][0]
gamenameFromFile = helper.getGamenameFromFilename(
romFile, romCollection)
artworkfiles = {}
artworkurls = []
self.createNfoFromDesc(
gamename, plot, romCollection.name, publisher, developer,
year, players, rating, votes, url, region, media,
perspective, controller, originalTitle, alternateTitle,
version, genreList, isFavorite, launchCount, romFile,
gamenameFromFile, artworkfiles, artworkurls)
progressDialog.writeMsg("", "", "", -1)
del progressDialog