def _parse_game_result(self, response):
""" response is expected to be a JSON object """
result = {}
# Standard fields
for k, v in self._game_mapping.items():
try:
# HACK - for compatibility we need to put each result in an array
#result[k] = response[v]
result[k] = [response[v]]
except KeyError as k:
log.warn("Unable to find key: {0}".format(k))
except Exception as e:
log.warn("Unable to extract data from key {0}".format(e))
# Custom fields (i.e. ones that require special handling
# HACK - for compatibility we need to put each result in an array
result['ReleaseYear'] = [self._parse_date(response['original_release_date'])]
result['Developer'] = self._parse_developers(response['developers'])
result['Publisher'] = self._parse_publishers(response['publishers'])
result['Genre'] = self._parse_genres(response['genres'])
# FIXME TODO Artwork and images are quite cumbersome to get from giantbomb search results
return result
def insertFile(self, fileName, gameId, fileType, romCollectionId, publisherId, developerId):
log.debug("Begin Insert file: %s" % fileName)
parentId = None
# TODO console and romcollection could be done only once per RomCollection
# fileTypeRow[3] = parent
if fileType.parent == 'game':
parentId = gameId
elif fileType.parent == 'romcollection':
parentId = romCollectionId
elif fileType.parent == 'publisher':
parentId = publisherId
elif fileType.parent == 'developer':
parentId = developerId
log.info("Inserting file with parent {0} (type {1})".format(parentId, fileType.parent))
fileRow = File(self.gdb).getFileByNameAndTypeAndParent(fileName, fileType.id, parentId)
if fileRow is None:
log.info("File does not exist in database. Insert file: %s" % fileName)
f = File(self.gdb)
try:
f.insert((fileName, fileType.id, parentId))
except Exception:
log.warn("Error inserting into database: %s" % fileName)
else:
log.info("File already exists in database: %s" % fileName)
def scanDescription(self, descFile, descParseInstruction, encoding):
Logutil.log('scanDescription: %s' % descFile, util.LOG_LEVEL_INFO)
if(descFile.startswith('http://')):
req = urllib2.Request(descFile)
req.add_unredirected_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31')
descFile = urllib2.urlopen(req).read()
else:
fh = open(str(descFile), 'r')
descFile = fh.read()
#load xmlDoc as elementtree to check with xpaths
tree = fromstring(descFile)
#single result as dictionary
result = {}
rootElement = self.grammarNode.attrib.get('root')
for node in tree.findall(rootElement):
result = self.parseElement(node)
result = self.replaceResultTokens(result)
yield result
def __init__(self):
log.info("init DBUpdate")
# self.scrapeResultsFile = ScrapeResultsLogFile()
self.missingDescFile = MissingDescLogFile()
self.missingArtworkFile = MissingArtworkLogFile()
self.possibleMismatchFile = MismatchLogFile()
def onClick(self, controlId):
log.debug("Begin onClick UIGameInfoView")
if (controlId == CONTROL_BUTTON_PLAYGAME):
self.launchEmu()
log.debug("End onClick UIGameInfoView")
def delete(self, gameId):
publisherId = self.getPublisherIdByGameId(gameId)
if publisherId != None:
obj = self.getObjectByQuery(self.publisherIdCountQuery, (publisherId,))
if (obj[0] < 2):
log.info("Delete Publisher with id %s" % str(publisherId))
self.deleteObjectByQuery(self.publisherDeleteQuery, (publisherId,))
def addScraperToSiteList(self, controlId, sites, romCollection):
log.info("addScraperToSiteList")
control = self.getControlById(controlId)
scraperItem = control.getSelectedItem()
scraper = scraperItem.getLabel()
if scraper == util.localize(32854):
return sites
#check if this site is already available for current RC
for site in romCollection.scraperSites:
if site.name == scraper:
sites.append(site)
return sites
siteRow = None
siteRows = self.gui.config.tree.findall('Scrapers/Site')
for element in siteRows:
if element.attrib.get('name') == scraper:
siteRow = element
break
if siteRow is None:
xbmcgui.Dialog().ok(util.localize(32021), util.localize(32022) % scraper)
return None
site, errorMsg = self.gui.config.readScraper(siteRow, romCollection.name, '', '', True, self.gui.config.tree)
if site is not None:
sites.append(site)
return sites
def rescrape_selection(self):
log.info("rescrape_selection")
romCollections = {}
listSize = self.gui.getListSize()
for i in range(0, listSize):
listItem = self.gui.getListItem(i)
romCollectionId = listItem.getProperty('romCollectionId')
try:
romCollection = romCollections[str(romCollectionId)]
except:
romCollection = self.gui.config.romCollections[str(
romCollectionId)]
romCollection.romPaths = []
files = File(self.gui.gdb).getRomsByGameId(
listItem.getProperty('gameId'))
try:
filename = files[0][0]
romCollection.romPaths.append(filename)
romCollections[romCollection.id] = romCollection
except:
log.info(
"Error getting filename for romCollectionId: {0}".format(
romCollectionId))
self.gui.rescrapeGames(romCollections)
def getArtworkForGame(self, romCollection, gamename, gamenameFromFile, gamedescription, foldername, publisher,
developer):
artWorkFound = False
artworkfiles = {}
artworkurls = {}
for path in romCollection.mediaPaths:
log.info("FileType: {0}".format(path.fileType.name))
# TODO replace %ROMCOLLECTION%, %PUBLISHER%, ...
fileName = path.path.replace("%GAME%", gamenameFromFile)
continueUpdate, artworkurls = self.getThumbFromOnlineSource(gamedescription, path.fileType.name,
fileName, artworkurls)
if not continueUpdate:
return False, {}, {}
log.debug("Additional data path: %s" % path.path)
files = self.resolvePath((path.path,), gamename, gamenameFromFile, foldername, romCollection.name,
publisher, developer)
if len(files) > 0:
artWorkFound = True
else:
self.missingArtworkFile.add_entry(gamename, gamenameFromFile, path.fileType.name)
artworkfiles[path.fileType] = files
return artWorkFound, artworkfiles, artworkurls
def insertGameFromDesc(self, gamedescription, gamename, romCollection,
filenamelist, foldername, isUpdate, gameId):
log.info("insertGameFromDesc")
if gamedescription is not None:
game = self.resolveParseResult(gamedescription, 'Game')
else:
game = ''
# if no game name has been scraped we expect that no results have been found
if game == '':
self.missingDescFile.add_entry(gamename)
if __addon__.getSetting(
util.SETTING_RCB_IGNOREGAMEWITHOUTDESC).upper() == 'TRUE':
log.warn(
"No description found for game '%s'. Game will not be imported."
% gamename)
return None
gamedescription = {}
gameId = self.insertData(gamedescription, gamename, romCollection,
filenamelist, foldername, isUpdate, gameId)
return gameId
def _parseSearchResults(self, response):
"""
Parse the json response from the Games/ByGameName API call
Returns a list of dicts with id, title and releaseDate
"""
results = []
code = response['code']
status = response['status']
if code != 200 or status != "Success":
log.error("thegamesdb returned an error. Code = %s, Status = %s" %
(code, status))
return results
data = response['data']
self.resultdata = {}
for result in data['games']:
results.append({
'id': result['id'],
'title': result['game_title'],
'releaseDate': result['release_date'],
'SearchKey': [result['game_title']]
})
#store result for later use in retrieve method
self.resultdata[result['id']] = result
log.info(u"Found {0} results using json response: {1}".format(
len(results), results))
return results
def get_scraper_by_name(self, sname):
"""Given a scraper name, returns the scraper class
Args:
sname: Name of the scraper, e.g. thegamesdb.net or MAME
Raises:
ConfigScraperSiteDoesNotExistException: No scraper matches the name
"""
try:
target = self.scrapers[sname]
except KeyError as e:
raise ConfigScraperSiteDoesNotExistException("Unsupported scraper: {0}".format(sname))
log.debug("Instantiating scraper class {0} - {1}".format(sname, target))
try:
module = __import__(target.lower())
class_ = getattr(module, target)
instance = class_()
except ImportError:
log.error("Unable to find scraper {0}".format(sname))
raise
return instance
def retrieve(self, gameid, platform=None):
result = {}
tree = ET.ElementTree()
if sys.version_info >= (2, 7):
parser = ET.XMLParser(encoding='utf-8')
else:
parser = ET.XMLParser()
tree.parse(self._get_xml_path(), parser)
#gameid is the exact name of the game used in element <description>
game = tree.find('.//game[description="%s"]'%gameid)
# Standard fields
for k, v in self._game_mapping.items():
# HACK - This is only used to retain backwards compatibility with existing scraper, where each key value was a
# list, even if only one value is in that list
try:
result[k] = [game.find(v).text]
# FIXME TODO When we remove the hack, this will be the code to use:
# result[k] = game.find(v).text
except Exception as e:
# Typically this result doesn't have this field
log.debug("Unable to extract data from key {0}".format(k))
# Custom fields
result['Genre'] = self._parse_genres(game)
return result
def get_scraper_by_name(self, sname):
"""Given a scraper name, returns the scraper class
Args:
sname: Name of the scraper, e.g. thegamesdb.net or MAME
Raises:
ConfigScraperSiteDoesNotExistException: No scraper matches the name
"""
try:
target = self.scrapers[sname]
except KeyError as e:
raise ConfigScraperSiteDoesNotExistException(
"Unsupported scraper: {0}".format(sname))
log.debug("Instantiating scraper class {0} - {1}".format(
sname, target))
try:
module = __import__(target.lower())
class_ = getattr(module, target)
instance = class_()
except ImportError:
log.error("Unable to find scraper {0}".format(sname))
raise
return instance
def add_romfiles_to_db(self, romFiles, gameId):
for romFile in romFiles:
log.debug("Adding romfile to DB: %s" % romFile)
fileType = FileType()
fileType.id, fileType.name, fileType.parent = 0, "rcb_rom", "game"
self.insertFile(romFile, gameId, fileType, None, None, None)
del fileType
def update_artwork_cache(self, console_id, file_type_id):
log.info('cache_artwork')
#cache all available artwork
media_dict = self._cache_mediapaths_for_selection(console_id, {})
if console_id > 0:
rom_collection = self.config.romCollections[str(console_id)]
self.dialogheader = util.localize(
32954) + " (%i / %i): %s" % (1, 1, rom_collection.name)
self._update_artwork_cache_for_romcollection(
rom_collection, file_type_id, media_dict)
else:
rccount = 1
rclen = len(self.config.romCollections)
for rcid in list(self.config.romCollections.keys()):
self.progress_dialog.itemCount = rclen
rom_collection = self.config.romCollections[str(rcid)]
self.dialogheader = util.localize(32954) + " (%i / %i): %s" % (
rccount, rclen, rom_collection.name)
continue_update = self._update_artwork_cache_for_romcollection(
rom_collection, file_type_id, media_dict)
if not continue_update:
break
rccount = rccount + 1
log.info('End cache_artwork')
def search(self, gamename, platform=None):
#use description to search for the game name as the name attribute also contains the region
#FIXME TODO
#currently not working with MAME xml files as the rom files don't use the friendly game name
pattern = "\<description\>(.*%s.*)\</description\>" % GameNameUtil(
).prepare_gamename_for_searchrequest(gamename)
results = []
try:
with io.open(self._get_xml_path(), 'r',
encoding="utf-8") as xmlfile:
for line in xmlfile:
result = re.search(pattern, line)
if result:
gamename = result.groups()[0]
results.append({
'id': gamename,
'title': gamename,
'SearchKey': [gamename]
})
except Exception as e:
log.error(e)
raise
return results
def open_json_url(self, **kwargs):
log.info('Retrieving url %s, params = %s' %
(kwargs['url'], kwargs['params']))
try:
r = requests.get(kwargs['url'],
headers=self._headers,
params=kwargs['params'])
except ValueError as e:
# Typically non-JSON response
raise ScraperUnexpectedContentException(
"Non-JSON response received")
log.debug(u"Retrieving {0} as JSON - HTTP{1}".format(
r.url, r.status_code))
if r.status_code == 401:
# Mobygames and GiantBomb send a 401 if the API key is invalid
raise ScraperUnauthorisedException("Invalid API key sent")
if r.status_code == 429:
raise ScraperExceededAPIQuoteException(
"Scraper exceeded API key limits")
if r.status_code == 500:
raise ScraperWebsiteUnavailableException("Website unavailable")
return r.json()
def updateSelectedRomCollection(self):
log.info("updateSelectedRomCollection")
self.addScrapersToSiteList()
# Image Placing Main
control = self.getControlById(CONTROL_LIST_IMAGEPLACING_MAIN)
imgPlacingItem = control.getSelectedItem()
imgPlacingName = imgPlacingItem.getLabel()
# HACK search key by value
for item in list(config.imagePlacingDict.items()):
if item[1] == imgPlacingName:
imgPlacingName = item[0]
imgPlacing, errorMsg = self.gui.config.readImagePlacing(imgPlacingName, self.gui.config.tree)
self.selectedRomCollection.imagePlacingMain = imgPlacing
# Image Placing Info
control = self.getControlById(CONTROL_LIST_IMAGEPLACING_INFO)
imgPlacingItem = control.getSelectedItem()
imgPlacingName = imgPlacingItem.getLabel()
# HACK search key by value
for item in list(config.imagePlacingDict.items()):
if item[1] == imgPlacingName:
imgPlacingName = item[0]
imgPlacing, errorMsg = self.gui.config.readImagePlacing(imgPlacingName, self.gui.config.tree)
self.selectedRomCollection.imagePlacingInfo = imgPlacing
# Update values for each of the buttons
for btn in self._control_buttons:
control = self.getControlById(btn['control'])
setattr(self.selectedRomCollection, btn['value'], bool(control.isSelected()))
def updateSelectedRomCollection(self):
log.info("updateSelectedRomCollection")
sites = []
sites = self.addScraperToSiteList(CONTROL_LIST_SCRAPER1, sites, self.selectedRomCollection)
self.selectedRomCollection.scraperSites = sites
# Image Placing Main
control = self.getControlById(CONTROL_LIST_IMAGEPLACING_MAIN)
imgPlacingItem = control.getSelectedItem()
imgPlacingName = imgPlacingItem.getLabel()
# HACK search key by value
for item in config.imagePlacingDict.items():
if item[1] == imgPlacingName:
imgPlacingName = item[0]
imgPlacing, errorMsg = self.gui.config.readImagePlacing(imgPlacingName, self.gui.config.tree)
self.selectedRomCollection.imagePlacingMain = imgPlacing
# Image Placing Info
control = self.getControlById(CONTROL_LIST_IMAGEPLACING_INFO)
imgPlacingItem = control.getSelectedItem()
imgPlacingName = imgPlacingItem.getLabel()
# HACK search key by value
for item in config.imagePlacingDict.items():
if item[1] == imgPlacingName:
imgPlacingName = item[0]
imgPlacing, errorMsg = self.gui.config.readImagePlacing(imgPlacingName, self.gui.config.tree)
self.selectedRomCollection.imagePlacingInfo = imgPlacing
# Update values for each of the buttons
for btn in self._control_buttons:
control = self.getControlById(btn['control'])
setattr(self.selectedRomCollection, btn['value'], bool(control.isSelected()))
def __get_archive_file_name(self, filename):
log.info("ArchiveHandler.__get_archive_file_name")
archive_file_name = 'archive://%(archive_file)s' % {
'archive_file': quote_plus(xbmcvfs.translatePath(filename))
}
log.debug("archive_file_name: {0}".format(archive_file_name))
return archive_file_name
def retrieve(self, gameid, platform):
result = {}
if not xbmcvfs.exists(self.nfo_file):
return result
fh = xbmcvfs.File(self.nfo_file)
game = ET.fromstring(fh.read())
fh.close()
# Standard fields
for k, v in self._game_mapping.items():
# HACK - This is only used to retain backwards compatibility with existing scraper, where each key value was a
# list, even if only one value is in that list
try:
result[k] = [game.find(v).text]
# FIXME TODO When we remove the hack, this will be the code to use:
# result[k] = game.find(v).text
except Exception:
# Typically this result doesn't have this field
log.debug("Unable to extract data from key {0}".format(k))
# Custom fields
result['Genre'] = self._parse_genres(game)
return result
def retrieve(self, gameid, platform):
result = {}
if not xbmcvfs.exists(self.nfo_file):
return result
game = ET.ElementTree()
if sys.version_info >= (2, 7):
parser = ET.XMLParser(encoding='utf-8')
else:
parser = ET.XMLParser()
game.parse(self.nfo_file, parser)
# Standard fields
for k, v in self._game_mapping.items():
# HACK - This is only used to retain backwards compatibility with existing scraper, where each key value was a
# list, even if only one value is in that list
try:
result[k] = [game.find(v).text]
# FIXME TODO When we remove the hack, this will be the code to use:
# result[k] = game.find(v).text
except Exception as e:
# Typically this result doesn't have this field
log.debug("Unable to extract data from key {0}".format(k))
# Custom fields
result['Genre'] = self._parse_genres(game)
return result
def _parseGameResult(self, game):
result = {}
# Standard fields
for k, v in self._game_mapping.items():
# HACK - This is only used to retain backwards compatibility with existing scraper, where each key value was a
# list, even if only one value is in that list
try:
result[k] = [game[v]]
except Exception:
# Typically this result doesn't have this field
log.debug("Unable to extract data from key {0}".format(k))
# Custom fields
# Adjust the date
releaseDate = game['release_date']
if releaseDate is not None:
result['ReleaseYear'] = [self._parse_date(releaseDate)]
result['Genre'] = self._parse_lookup_data(
game['genres'], self.genres['data']['genres'])
result['Developer'] = self._parse_lookup_data(
game['developers'], self.developers['data']['developers'])
result['Publisher'] = self._parse_lookup_data(
game['publishers'], self.publishers['data']['publishers'])
"""
# Prefix images with base url
for image in ['fanart', 'boxfront', 'boxback', 'screenshot', 'clearlogo']:
try:
result['Filetype' + image] = ["http://thegamesdb.net/banners/" + result['Filetype' + image][0]]
except KeyError:
log.warn("Image type {0} not present in retrieve results".format(image))
"""
return result
def __audioSuspend(self):
if __addon__.getSetting(
util.SETTING_RCB_SUSPENDAUDIO).upper() == 'TRUE':
log.debug("Suspending audio")
xbmc.executebuiltin("PlayerControl(Stop)")
xbmc.enableNavSounds(False)
xbmc.audioSuspend()
def add_romfiles_to_db(self, romFiles, gameId):
for romFile in romFiles:
log.debug("Adding romfile to DB: %s" % romFile)
fileType = FileType()
fileType.id, fileType.name, fileType.parent = 0, "rcb_rom", "game"
self.insertFile(romFile, gameId, fileType, None, None, None)
del fileType
def _parse_search_results(self, response):
""" response is expected to be a JSON object """
log.debug("Parsing response for search results: {0}".format(response))
results = []
if response['number_of_total_results'] == 0:
log.warn("No results found")
return results
for result in response['results']:
try:
year = self._parse_date(result['release_date'])
results.append({
'id': result['guid'],
'title': result['name'],
'releaseDate': year,
'SearchKey': [result['name']]
})
except KeyError:
log.warn("Unable to find expected field in response")
except Exception as e:
log.warn("Error parsing field: {0}".format(e))
log.debug("Found {0} results using requests JSON parser: {1}".format(
len(results), results))
return results
def insertGame(self, game_row, isUpdate, gameId, allowUpdate):
#HACK: delete first element as we do not want to insert or update the id
del game_row[0]
# Check if exists and insert/update as appropriate; move this functionality to the Game object
try:
if not isUpdate:
log.info(u"Game does not exist in database. Insert game: %s" % game_row[DataBaseObject.COL_NAME])
Game(self.gdb).insert(game_row)
return self.gdb.cursor.lastrowid
else:
if allowUpdate:
# check if we are allowed to update with null values
allowOverwriteWithNullvalues = __addon__.getSetting(
util.SETTING_RCB_ALLOWOVERWRITEWITHNULLVALUES).upper() == 'TRUE'
log.info("allowOverwriteWithNullvalues: {0}".format(allowOverwriteWithNullvalues))
log.info(u"Game does exist in database. Update game: %s" % game_row[DataBaseObject.COL_NAME])
#remove id from column list
columns = Game.FIELDNAMES[1:]
Game(self.gdb).update(columns, game_row, gameId, allowOverwriteWithNullvalues)
else:
log.info(
u"Game does exist in database but update is not allowed for current rom collection. game: %s" % game_row[DataBaseObject.COL_NAME])
return gameId
except Exception as exc:
log.error(u"An error occured while adding game '%s'. Error: %s" % (game_row[DataBaseObject.COL_NAME], exc))
return None
def _parse_game_result(self, response):
""" response is expected to be a JSON object """
result = {}
# Standard fields
for k, v in self._game_mapping.items():
try:
# HACK - for compatibility we need to put each result in an array
#result[k] = response[v]
result[k] = [response[v]]
except KeyError as k:
log.warn("Unable to find key: {0}".format(k))
except Exception as e:
log.warn("Unable to extract data from key {0}".format(e))
# Custom fields (i.e. ones that require special handling
# HACK - for compatibility we need to put each result in an array
result['ReleaseYear'] = [
self._parse_date(response['original_release_date'])
]
result['Developer'] = self._parse_developers(response['developers'])
result['Publisher'] = self._parse_publishers(response['publishers'])
result['Genre'] = self._parse_genres(response['genres'])
# FIXME TODO Artwork and images are quite cumbersome to get from giantbomb search results
return result
def addRomCollection(self, configObj):
Logutil.log("Begin addRomCollection", util.LOG_LEVEL_INFO)
consoleList = sorted(WebScraper().consoleDict.keys())
new_id = 1
rcIds = configObj.romCollections.keys()
rcIds.sort()
#read existing rom collection ids and names
for rcId in rcIds:
#remove already configured consoles from the list
if configObj.romCollections[rcId].name in consoleList:
consoleList.remove(configObj.romCollections[rcId].name)
#find highest id
if int(rcId) > int(new_id):
new_id = rcId
new_id = int(new_id) + 1
# Add new rom collections
success, romCollections = self.addRomCollections(new_id, configObj, consoleList, True)
if not success:
log.info("Action canceled. Config.xml will not be written")
return False, util.localize(32172)
# Update config file
configWriter = ConfigXmlWriter(False)
success, message = configWriter.writeRomCollections(romCollections, False)
#create artwork directories for all rom collections
helper.createArtworkDirectories(romCollections)
log.info("End addRomCollection")
return success, message
def readImagePlacing(self, imagePlacingName, tree):
#fileTypeForRow = None
fileTypeForRows = tree.findall('ImagePlacing/fileTypeFor')
fileTypeForRow = next((element for element in fileTypeForRows if element.attrib.get('name') == imagePlacingName), None)
if fileTypeForRow is None:
Logutil.log('Configuration error. ImagePlacing/fileTypeFor %s does not exist in config.xml' % str(imagePlacingName), util.LOG_LEVEL_ERROR)
return None, util.localize(32005)
imagePlacing = ImagePlacing()
imagePlacing.name = imagePlacingName
for attr in ['fileTypesForGameList', 'fileTypesForGameListSelected',
'fileTypesForMainView1', 'fileTypesForMainView2', 'fileTypesForMainView3',
'fileTypesForMainViewBackground', 'fileTypesForMainViewGameInfoBig',
'fileTypesForMainViewGameInfoUpperLeft', 'fileTypesForMainViewGameInfoUpperRight',
'fileTypesForMainViewGameInfoLowerLeft', 'fileTypesForMainViewGameInfoLowerRight',
'fileTypesForMainViewGameInfoLower', 'fileTypesForMainViewGameInfoUpper',
'fileTypesForMainViewGameInfoRight', 'fileTypesForMainViewGameInfoLeft',
'fileTypesForMainViewVideoWindowBig', 'fileTypesForMainViewVideoWindowSmall',
'fileTypesForMainViewVideoFullscreen']:
# Hack - class attribute fileTypesForXXX doesn't match XML key fileTypeForXXX
val = self.readFileTypeForElement(fileTypeForRow, attr.replace('fileTypesFor', 'fileTypeFor'), tree)
log.debug("Reading imageplacing for {0}: {1}".format(attr, val))
setattr(imagePlacing, attr, val)
return imagePlacing, ''
def retrieve(self, gameid, platform=None):
result = {}
fh = xbmcvfs.File(self._get_xml_path())
tree = ET.fromstring(fh.read())
fh.close()
#gameid is the exact name of the game used in <description> or @name (for platforms with MAME style names)
searchpattern = './/game[description="%s"]' % gameid
if platform in self._MAME_style_platforms:
searchpattern = './/game[@name="%s"]' % gameid
game = tree.find(searchpattern)
# Standard fields
for k, v in self._game_mapping.items():
# HACK - This is only used to retain backwards compatibility with existing scraper, where each key value was a
# list, even if only one value is in that list
try:
result[k] = [game.find(v).text]
# FIXME TODO When we remove the hack, this will be the code to use:
# result[k] = game.find(v).text
except Exception:
# Typically this result doesn't have this field
log.debug("Unable to extract data from key {0}".format(k))
# Custom fields
result['Genre'] = self._parse_genres(game)
return result
def __init__(self, *args, **kwargs):
# Don't put GUI sensitive stuff here (as the xml hasn't been read yet)
log.info("init ContextMenu")
self.gui = kwargs["gui"]
self.doModal()
def delete(self, gameId):
yearId = self.getYearIdByGameId(gameId)
if yearId != None:
obj = self.getObjectByQuery(self.yearIdCountQuery, (yearId,))
if obj[0] < 2:
log.info("Delete Year with id %s" % str(yearId))
self.deleteObjectByQuery(self.yearDeleteQuery, (yearId,))
def buildMediaTypeList(self, configObj, isUpdate):
#build fileTypeList
fileTypeList = []
if isUpdate:
fileTypes = configObj.tree.findall('FileTypes/FileType')
else:
#build fileTypeList
configFile = util.joinPath(util.getAddonInstallPath(), 'resources',
'database', 'config_template.xml')
if not xbmcvfs.exists(configFile):
log.error(
"File config_template.xml does not exist. Place a valid config file here: "
+ str(configFile))
return None, util.localize(32040)
tree = ElementTree().parse(configFile)
fileTypes = tree.findall('FileTypes/FileType')
for fileType in fileTypes:
name = fileType.attrib.get('name')
if name != None:
mediaType = fileType.find('type')
if mediaType != None and mediaType.text == 'video':
name = name + ' (video)'
fileTypeList.append(name)
return fileTypeList, ''
def delete(self, gameId):
developerId = self.getDeveloperIdByGameId(gameId)
if developerId != None:
obj = self.getObjectByQuery(self.developerIdCountQuery, (developerId,))
if obj[0] < 2:
log.info("Delete Developer with id %s" % str(developerId))
self.deleteObjectByQuery(self.developerDeleteQuery, (developerId,))
def search(self, gamename, platform=None):
#use description to search for the game name as the name attribute also contains the region
pattern = "\<description\>(.*%s.*)\</description\>" % GameNameUtil(
).prepare_gamename_for_searchrequest(gamename)
if platform in self._MAME_style_platforms:
pattern = 'name="(.*%s.*)"' % GameNameUtil(
).prepare_gamename_for_searchrequest(gamename)
results = []
try:
fh = xbmcvfs.File(self._get_xml_path())
text = fh.read()
text = util.html_unescape(text)
fh.close()
for line in text.splitlines():
#HACK: Apostrophes are removed in prepare_gamename_for_searchrequest. So we also have to do it here.
result = re.search(pattern, line.replace("'", ""))
if result:
gamename = result.groups()[0]
results.append({
'id': gamename,
'title': gamename,
'SearchKey': [gamename]
})
except Exception as e:
log.error(e)
raise
return results
def search(self, gamename, platform=None):
""" Ignore platform """
# Newer versions support multi systems, $info=XXX indicates an arcade ROM
startmarker = "$info=%s," %gamename
gamedata = False
data = ""
historyfile_path = self._get_history_path()
try:
with io.open(historyfile_path, 'r', encoding="utf-8") as historyfile:
for line in historyfile:
if line.startswith(startmarker):
gamedata = True
if gamedata:
data += line
if line.startswith("$end"):
gamedata = False
except Exception as e:
log.error(e)
raise
try:
# Note the regex has to search for either Windows-style line endings (\r\n) or Unix-style (\n)
# Earlier history.dat files had 3 line breaks, newer versions have 2
# We also rename manufacturer and Publisher, and Description is all data between the bio heading and the first
# subheading (e.g. - TECHNICAL - or - CONTRIBUTE -). The romset entry is delimited by the $end.
# Japanese (or other foreign titles) have the translated name in brackets underneath.
# Newer versions have an age-based reference (e.g. A 24-year-old SNK Neo-Geo MVS Cart) between the $bio
# and title line
pattern = r"\$bio(\r?\n){2}" \
"(?P<AgeRef>.*?(\r?\n){2})?" \
"(?P<Game>.*?) \(c\) (?P<ReleaseYear>\d{4}) (?P<Publisher>.*?)\.(\r?\n)" \
"(\((?P<Translation>.*?)\))?(\r?\n){1,2}" \
"(?P<Description>.*?)(\r?\n){2,3}" \
"- [A-Z]"
rdata = re.search(pattern, data, re.DOTALL | re.MULTILINE | re.UNICODE)
if rdata is None:
raise ScraperNoSearchResultsFoundException("Unable to find %s in MAME history dat file" %gamename)
except Exception as e:
print "Error searching for game %s using regex: %s" %(gamename, str(e))
return []
self.resultdata = [rdata.groupdict()]
self.resultdata[0]['id'] = self.resultdata[0]['Game']
self.resultdata[0]['SearchKey'] = self.resultdata[0]['Game']
# HACK - This is only used to retain backwards compatibility with existing scraper, where each key value was a
# list, even if only one value is in that list
for k, v in self.resultdata[0].iteritems():
self.resultdata[0][k] = [v]
return self.resultdata
def walkDownPath(self, files, romPath, maxFolderDepth):
log.info("alkDownPath romPath: %s" % romPath)
files = self.walkDown(files, romPath, maxFolderDepth)
log.info("files after walkDown = %s" % files)
return files
def updateControls(self):
log.info('updateControls')
control = self.getControlById(CONTROL_LIST_ROMCOLLECTIONS)
selectedRomCollectionName = str(control.getSelectedItem().getLabel())
self.selectedRomCollection = self.gui.config.getRomCollectionByName(selectedRomCollectionName)
def add_genres_to_db(self, genreIds, gameId):
# If the genre-game link doesn't exist in the DB, create it
for genreId in genreIds:
genreGame = GenreGame(self.gdb).getGenreGameByGenreIdAndGameId(genreId, gameId)
if genreGame is None:
log.debug("Inserting link between game %s and genre %s" % (str(gameId), str(genreId)))
GenreGame(self.gdb).insert((genreId, gameId))
del genreGame
def __init__(self):
self.env = (os.environ.get("OS", "win32"), "win32", )[os.environ.get("OS", "win32") == "xbox"]
log.debug("Running environment detected as {0}".format(self.env))
# Do we need to escape commands before executing?
self.escapeCmd = __addon__.getSetting(util.SETTING_RCB_ESCAPECOMMAND).upper() == 'TRUE'
self.romCollection = None
def selectScrapersInList(self, sitesInRomCollection, sitesInList):
log.info("selectScrapersInList")
if len(sitesInRomCollection) >= 1:
self.selectItemInList(sitesInRomCollection[0].name, CONTROL_LIST_SCRAPER1)
else:
self.selectItemInList(util.localize(32854), CONTROL_LIST_SCRAPER1)
def __init__(self, *args, **kwargs):
log.info("init Edit Rom Collection")
self.gui = kwargs["gui"]
self.romCollections = self.gui.config.romCollections
self.scraperSites = self.gui.config.scraperSites
self.doModal()
def open_xml_url(self, **kwargs):
log.info('Retrieving url %s, params = %s' %(kwargs['url'], kwargs['params']))
r = requests.get(kwargs['url'], headers=self._headers, params=kwargs['params'])
log.debug(u"Retrieving {0} as XML - HTTP{1}".format(r.url, r.status_code))
# Need to ensure we are sending back Unicode text
return r.text.encode('utf-8')
def __getEncoding(self):
# HACK: sys.getfilesystemencoding() is not supported on all systems (e.g. Android)
try:
encoding = sys.getfilesystemencoding()
except Exception as e:
log.warn("Unable to get filesystem encoding, defaulting to UTF-8")
encoding = 'utf-8'
return encoding
def selectItemInList(self, options, itemName, controlId):
log.info('selectItemInList')
for i in range(0, len(options)):
option = options[i]
if itemName == option:
control = self.getControlById(controlId)
control.selectItem(i)
break
def findFilesByGameDescription(self, key, fileDict):
log.info("searching for Key: %s" % key)
try:
filename = fileDict[key]
log.info("result found: %s" % filename)
except KeyError:
filename = None
return filename
def compareNames(self, gamename, searchkey, checkSubtitle):
if checkSubtitle:
if searchkey.find(gamename) > -1:
log.info("%s is a subtitle of %s. Using result %s" % (gamename, searchkey, searchkey))
return True
else:
if gamename == searchkey:
log.info("Perfect match. Using result %s" % searchkey)
return True
return False
def onInit(self):
log.info('onInit Remove Rom Collection')
# Rom Collections
log.info('build rom collection list')
self.addItemsToList(CONTROL_LIST_ROMCOLLECTIONS, self.gui.config.getRomCollectionNames())
# Delete Options
rcDeleteOptions = [util.localize(32137), util.localize(32138)]
self.addItemsToList(CONTROL_LIST_DELETEOPTIONS, rcDeleteOptions, properties=['RCollection', 'Roms'])
self.updateControls()
def resolveParseResult(self, result, itemName):
resultValue = u''
try:
resultValue = result[itemName][0].strip()
if type(resultValue) == str:
resultValue = resultValue.decode('utf-8')
except Exception, (exc):
log.warn(u"Error while resolving item: %s: %s" % (itemName, exc))
def getScrapingMode(self):
mode = 0
scrape_options = {util.SCRAPING_OPTION_AUTO_ACCURATE_TXT: 0,
util.SCRAPING_OPTION_INTERACTIVE_TXT: 1}
try:
mode = scrape_options[__addon__.getSetting(util.SETTING_RCB_SCRAPINGMODE)]
except KeyError:
pass
log.info("Scraping mode: {0}".format(mode))
return mode
def walkDown(self, files, romPath, maxFolderDepth):
log.info("Running walkdown on: %s" % romPath)
dirs, newFiles, dirname, filemask = self.getFilesByWildcardExt(romPath)
files.extend(newFiles)
for dir in dirs:
newRomPath = util.joinPath(dirname, dir, filemask)
maxFolderDepth = maxFolderDepth - 1
if (maxFolderDepth > 0):
self.walkDown(files, newRomPath, maxFolderDepth)
return files
def _parse_screenshots_result(self, response):
result = {}
screenshots = response['screenshots']
if len(screenshots) == 0:
log.warn("No screenshots found in mobygames response")
return result
#HACK: always return the first screenshot. We could add support for multiple screenshots or screenshot selection.
screenshot = screenshots[0]
result['Filetypescreenshot'] = [screenshot['image']]
return result
def insertFile(self, fileName, gameId, fileType, romCollectionId, publisherId, developerId):
log.debug("Begin Insert file: %s" % fileName)
parentId = None
# TODO console and romcollection could be done only once per RomCollection
# fileTypeRow[3] = parent
if fileType.parent == 'game':
parentId = gameId
elif fileType.parent == 'romcollection':
parentId = romCollectionId
elif fileType.parent == 'publisher':
parentId = publisherId
elif fileType.parent == 'developer':
parentId = developerId
log.info("Inserting file with parent {0} (type {1})".format(parentId, fileType.parent))
fileRow = File(self.gdb).getFileByNameAndTypeAndParent(fileName, fileType.id, parentId)
if fileRow is None:
log.info("File does not exist in database. Insert file: %s" % fileName)
f = File(self.gdb)
try:
f.insert((fileName, fileType.id, parentId))
except Exception, (exc):
log.warn("Error inserting into database: %s" % fileName)
def __handleCompressedFile(self, gui, filext, rom, emuParams):
log.info("__handleCompressedFile")
# Note: Trying to delete temporary files (from zip or 7z extraction) from last run
# Do this before launching a new game. Otherwise game could be deleted before launch
tempDir = os.path.join(util.getTempDir(), 'extracted', self.romCollection.name)
# check if folder exists
if not xbmcvfs.exists(tempDir +'\\'):
log.info("Create temporary folder: " +tempDir)
xbmcvfs.mkdir(tempDir)
try:
if xbmcvfs.exists(tempDir +'\\'):
log.info("Trying to delete temporary rom files")
#can't use xbmcvfs.listdir here as it seems to cache the file list and RetroPlayer won't find newly created files anymore
files = os.listdir(tempDir)
for f in files:
#RetroPlayer places savestate files next to the roms. Don't delete these files.
fname, ext = os.path.splitext(f)
if(ext not in ('.sav', '.xml', '.png')):
xbmcvfs.delete(os.path.join(tempDir, f))
except Exception, (exc):
log.error("Error deleting files after launch emu: " + str(exc))
gui.writeMsg(util.localize(32036) + ": " + str(exc))
def _parse_date(self, datestr):
"""Extract the year from a given date string using a given format. This function is used to cater for
an edge case identified in https://forum.kodi.tv/showthread.php?tid=112916&pid=1214507#pid1214507.
Args:
datestr: Input date
Returns:
Year as a %Y format string
"""
if datestr is None:
return '1970'
x = None
for fmt2 in ["%Y-%m-%d", "%Y-%m", "%Y", "%Y-%m-%d %H:%M:%S", "%d/%m/%Y", "%m/%d/%Y"]:
try:
x = datetime.strptime(datestr, fmt2).strftime("%Y")
break
except ValueError as e:
# Skip to the next format
log.warn("ValueError in parseDate: %s" %e)
except TypeError as e:
log.warn("Unable to parse date using strptime, falling back to time function")
try:
x = datetime(*(time.strptime(datestr, fmt2)[0:6])).strftime("%Y")
break
except ValueError as ve:
log.warn("Unable to parse date %s using %s, try next format. %s" %(datestr, fmt2, ve))
if x is not None:
return x
else:
log.warn(u"Unexpected date format: {0}".format(datestr))
return u"1970"
def __executeCommand(self, cmd):
# change working directory
path = os.path.dirname(self.romCollection.emulatorCmd)
if os.path.isdir(path):
try:
os.chdir(path)
except OSError:
log.warn("Unable to chdir to {0}".format(path))
if self.romCollection.usePopen:
import subprocess
process = subprocess.Popen(cmd.encode(self.__getEncoding()), shell=True)
process.wait()
else:
os.system(cmd.encode(self.__getEncoding()))
