def createSets(self, dats): self.logger.log('Creating or cleaning output directory ' + self.configuration['exportDir']) if os.path.exists(self.configuration['exportDir']): for file in os.listdir( os.path.join(self.configuration['exportDir'])): fullPath = os.path.join(self.configuration['exportDir'], file) shutil.rmtree(fullPath) if os.path.isdir( fullPath) else os.remove(fullPath) else: os.makedirs(self.configuration['exportDir']) dryRun = True if self.configuration['dryRun'] == '1' else False useGenreSubFolder = True if self.configuration[ 'genreSubFolders'] == '1' else False scrapeImages = True if self.configuration[ 'useImages'] == '1' and self.configuration['images'] else False CSVs, gamelists, roots = dict(), dict(), dict() header = "Genre;Name (mame description);Rom name;Year;Manufacturer;Hardware;Comments;Notes\n" # init CSVS CSVs[self.setKey] = open(os.path.join(self.configuration['exportDir'], self.setKey + ".csv"), "w", encoding="utf-8") CSVs[self.setKey].write(header) # init gamelists roots[self.setKey] = etree.Element("datafile") roots[self.setKey].append(dats[self.setKey + "Header"]) os.makedirs(os.path.join(self.configuration['exportDir'], self.setKey)) os.makedirs( os.path.join(self.configuration['exportDir'], self.setKey, 'downloaded_images')) if scrapeImages else None gamelists[self.setKey] = gamelist.initWrite( os.path.join(self.configuration['exportDir'], self.setKey)) for genre in self.favorites.keys(): self.logger.log("Handling genre " + genre) if useGenreSubFolder: os.makedirs( os.path.join(self.configuration['exportDir'], self.setKey, genre)) if scrapeImages: gamelist.writeGamelistFolder(gamelists[self.setKey], genre, genre + '.png') utils.setImageCopy( self.configuration['exportDir'], os.path.join(self.scriptDir, 'data', 'images'), genre + '.png', self.setKey, dryRun) # copy bios in each subdirectory for bios in self.bioses: setBios = os.path.join(self.configuration[self.setKey], bios + ".zip") utils.setFileCopy(self.configuration['exportDir'], setBios, genre, bios, self.setKey, useGenreSubFolder, dryRun) if os.path.exists(setBios): utils.writeGamelistHiddenEntry(gamelists[self.setKey], bios, genre, useGenreSubFolder) for game in sorted(self.favorites[genre]): setRom = os.path.join(self.configuration[self.setKey], game + ".zip") setCHD = os.path.join(self.configuration[self.setKey], game) image = self.configuration['imgNameFormat'].replace( '{rom}', game) # TODO aliases should be handled here utils.setFileCopy(self.configuration['exportDir'], setRom, genre, game, self.setKey, useGenreSubFolder, dryRun) utils.setCHDCopy(self.configuration['exportDir'], setCHD, genre, game, self.setKey, useGenreSubFolder, dryRun) utils.writeCSV(CSVs[self.setKey], game, None, genre, dats[self.setKey], None, self.setKey) utils.writeGamelistEntry(gamelists[self.setKey], game, image, dats[self.setKey], genre, useGenreSubFolder, None, self.setKey, None) roots[self.setKey].append( dats[self.setKey][game].node) if game in dats[ self.setKey] else None if scrapeImages: utils.setImageCopy(self.configuration['exportDir'], self.configuration['images'], image, self.setKey, dryRun) self.logger.log(setRom) # writing and closing everything treeSet = etree.ElementTree(roots[self.setKey]) treeSet.write(os.path.join(self.configuration['exportDir'], self.setKey + ".dat"), xml_declaration=True, encoding="utf-8") CSVs[self.setKey].close() gamelist.closeWrite(gamelists[self.setKey])
def createSets(self, allTests, dats): self.logger.log('Creating or cleaning output directory ' + self.configuration['exportDir']) if os.path.exists(self.configuration['exportDir']): for file in os.listdir( os.path.join(self.configuration['exportDir'])): fullPath = os.path.join(self.configuration['exportDir'], file) shutil.rmtree(fullPath) if os.path.isdir( fullPath) else os.remove(fullPath) else: os.makedirs(self.configuration['exportDir']) notInAnySet = [] onlyInOneSet = dict() dryRun = True if self.configuration['dryRun'] == '1' else False useGenreSubFolder = True if self.configuration[ 'genreSubFolders'] == '1' else False keepNotTested = True if self.configuration[ 'keepNotTested'] == '1' else False keepLevel = int(self.configuration['keepLevel']) usePreferedSetForGenre = True if self.configuration[ 'usePreferedSetForGenre'] == '1' else False scrapeImages = True if self.configuration[ 'useImages'] == '1' and self.configuration['images'] else False scoreSheet = open(os.path.join(self.configuration['exportDir'], "scoreSheet.csv"), "w", encoding="utf-8") scoreSheet.write( 'rom;fbneoScore;mame2003Score;mame2003PlusScore;mame2010Score\n') CSVs, gamelists, roots = dict(), dict(), dict() header = "Status;Genre;Name (mame description);Rom name;Year;Manufacturer;Hardware;Comments;Notes\n" for setKey in self.usingSystems: # init CSVS CSVs[setKey] = open(os.path.join(self.configuration['exportDir'], setKey + ".csv"), "w", encoding="utf-8") CSVs[setKey].write(header) # init gamelists roots[setKey] = etree.Element("datafile") roots[setKey].append(dats[setKey + "Header"]) os.makedirs(os.path.join(self.configuration['exportDir'], setKey)) os.makedirs( os.path.join(self.configuration['exportDir'], setKey, 'downloaded_images')) if scrapeImages else None gamelists[setKey] = gamelist.initWrite( os.path.join(self.configuration['exportDir'], setKey)) for genre in self.favorites.keys(): self.logger.log("Handling genre " + genre) if useGenreSubFolder: for setKey in self.usingSystems: os.makedirs( os.path.join(self.configuration['exportDir'], setKey, genre)) if scrapeImages: gamelist.writeGamelistFolder(gamelists[setKey], genre, genre + '.png') self.setImageCopy( os.path.join(self.scriptDir, 'data', 'images'), genre + '.png', setKey, dryRun) # copy bios in each subdirectory for bios in self.bioses: for setKey in self.usingSystems: setBios = os.path.join(self.configuration[setKey], bios + ".zip") self.setFileCopy(setBios, genre, bios, setKey, useGenreSubFolder, dryRun) if os.path.exists(setBios): self.writeGamelistHiddenEntry(gamelists[setKey], bios, genre, useGenreSubFolder) for game in sorted(self.favorites[genre]): audit = game + " -> " scores = dict() testForGame = allTests[game] if game in allTests else None for setKey in self.setKeys: scores[setKey] = self.computeScore( setKey, self.configuration[setKey], game, testForGame) if setKey in self.usingSystems else -2 audit = audit + " SCORES: " + str( scores[self.fbneoKey]) + " " + str( scores[self.mame2003Key]) + " " + str( scores[self.mame2003plusKey]) + " " + str( scores[self.mame2010Key]) + " ," scoreSheet.write( '%s;%i;%i;%i;%i\n' % (game, scores[self.fbneoKey], scores[self.mame2003Key], scores[self.mame2003plusKey], scores[self.mame2010Key])) selected = [] for setKey in self.usingSystems: selected.append(setKey) if self.keepSet( keepNotTested, usePreferedSetForGenre, self.configuration['exclusionType'], keepLevel, scores, setKey, genre, selected) else None audit = audit + " SELECTED: " + str(selected) for setKey in self.usingSystems: setRom = os.path.join(self.configuration[setKey], game + ".zip") image = self.configuration['imgNameFormat'].replace( '{rom}', game) if setKey in selected: # TODO aliases should be handled here self.setFileCopy(setRom, genre, game, setKey, useGenreSubFolder, dryRun) self.writeCSV(CSVs[setKey], game, scores[setKey], genre, dats[setKey], testForGame, setKey) self.writeGamelistEntry(gamelists[setKey], game, image, dats[setKey], genre, useGenreSubFolder, testForGame, setKey) roots[setKey].append( dats[setKey] [game].node) if game in dats[setKey] else None if scrapeImages: self.setImageCopy(self.configuration['images'], image, setKey, dryRun) if len(selected) == 0: notInAnySet.append(game) elif len(selected) == 1: if selected[0] not in onlyInOneSet: onlyInOneSet[selected[0]] = [] onlyInOneSet[selected[0]].append(game) self.logger.log(" " + audit) # writing and closing everything for setKey in self.usingSystems: treeSet = etree.ElementTree(roots[setKey]) treeSet.write(os.path.join(self.configuration['exportDir'], setKey + ".dat"), xml_declaration=True, encoding="utf-8") CSVs[setKey].close() gamelist.closeWrite(gamelists[setKey]) scoreSheet.close() self.logger.log("\n<------------------ RESULTS ------------------>") self.logger.log("NOT FOUND IN ANY SET : " + str(len(notInAnySet))) self.logger.logList("", notInAnySet)
def __createSets(self, allTests, dats): self.logger.log('Creating or cleaning output directory ' + self.configuration['exportDir']) if os.path.exists(self.configuration['exportDir']): for file in os.listdir( os.path.join(self.configuration['exportDir'])): fullPath = os.path.join(self.configuration['exportDir'], file) shutil.rmtree(fullPath) if os.path.isdir( fullPath) else os.remove(fullPath) else: os.makedirs(self.configuration['exportDir']) notInAnySet = [] onlyInOneSet = dict() dryRun = True if self.configuration['dryRun'] == '1' else False useGenreSubFolder = True if self.configuration[ 'genreSubFolders'] == '1' else False keepNotTested = True if self.configuration[ 'keepNotTested'] == '1' else False keepLevel = int(self.configuration['keepLevel']) usePreferedSetForGenre = True if self.configuration[ 'usePreferedSetForGenre'] == '1' else False scrapeImages = True if self.configuration[ 'useImages'] == '1' and self.configuration['images'] else False scoreSheet = open(os.path.join(self.configuration['exportDir'], "scoreSheet.csv"), "w", encoding="utf-8") scoreSheet.write('rom;' + ';'.join( list(map(lambda key: key + 'Score', self.setKeys[self.hardware]))) + '\n') CSVs, gamelists, roots = dict(), dict(), dict() header = "Status;Genre;Name (mame description);Rom name;Year;Manufacturer;Hardware;Comments;Notes\n" for setKey in self.usingSystems: # init CSVS CSVs[setKey] = open(os.path.join(self.configuration['exportDir'], setKey + ".csv"), "w", encoding="utf-8") CSVs[setKey].write(header) # init gamelists roots[setKey] = etree.Element("datafile") roots[setKey].append(dats[setKey + "Header"]) os.makedirs(os.path.join(self.configuration['exportDir'], setKey)) os.makedirs( os.path.join(self.configuration['exportDir'], setKey, 'downloaded_images')) if scrapeImages else None gamelists[setKey] = gamelist.initWrite( os.path.join(self.configuration['exportDir'], setKey)) # get bioses if '[BIOSES]' in self.favorites.keys(): self.bioses = self.favorites['[BIOSES]'] del self.favorites['[BIOSES]'] for genre in self.favorites.keys(): self.logger.log("Handling genre " + genre) if useGenreSubFolder: for setKey in self.usingSystems: os.makedirs( os.path.join(self.configuration['exportDir'], setKey, genre)) if scrapeImages: gamelist.writeGamelistFolder(gamelists[setKey], genre, genre + '.png') utils.setImageCopy( self.configuration['exportDir'], os.path.join(self.scriptDir, 'data', 'images'), genre + '.png', setKey, dryRun) # copy bios in each subdirectory for bios in self.bioses: for setKey in self.usingSystems: setBios = os.path.join(self.configuration[setKey], bios + ".zip") utils.setFileCopy(self.configuration['exportDir'], setBios, genre, bios, setKey, useGenreSubFolder, dryRun) if os.path.exists(setBios): utils.writeGamelistHiddenEntry(gamelists[setKey], bios, genre, useGenreSubFolder) for favs in sorted(self.favorites[genre]): # needed to handle multi names games if ';' in favs: games = favs.split(';') else: games = [favs] multiGameFoundInSet = False for game in games: audit = game + " -> " scores = dict() testForGame = allTests[game] if game in allTests else None for setKey in self.setKeys[self.hardware]: scores[setKey] = self.__computeScore( setKey, self.configuration[setKey], game, testForGame) if setKey in self.usingSystems else -2 audit = audit + " SCORES: " + \ " ".join(list(map(lambda key: str(scores[key]), self.setKeys[self.hardware]))) + " ," scoreSheet.write(game + ';' + ';'.join( list( map(lambda key: str(scores[key]), self.setKeys[ self.hardware]))) + '\n') selected = [] for setKey in self.usingSystems: selected.append(setKey) if self.__keepSet( keepNotTested, usePreferedSetForGenre, self.configuration['exclusionType'], keepLevel, scores, setKey, genre, selected) else None audit = audit + " SELECTED: " + str(selected) for setKey in self.usingSystems: setRom = os.path.join(self.configuration[setKey], game + ".zip") setCHD = os.path.join(self.configuration[setKey], game) image = self.configuration['imgNameFormat'].replace( '{rom}', game) if setKey in selected: multiGameFoundInSet = True utils.setFileCopy(self.configuration['exportDir'], setRom, genre, game, setKey, useGenreSubFolder, dryRun) utils.setCHDCopy(self.configuration['exportDir'], setCHD, genre, game, setKey, useGenreSubFolder, dryRun) utils.writeCSV(CSVs[setKey], game, scores[setKey], genre, dats[setKey], testForGame, setKey) testStatus = self.getStatus(testForGame[setKey].status) \ if testForGame is not None and setKey in testForGame else 'UNTESTED & FRESHLY ADDED' utils.writeGamelistEntry(gamelists[setKey], game, image, dats[setKey], genre, useGenreSubFolder, testForGame, setKey, testStatus) roots[setKey].append( dats[setKey] [game].node) if game in dats[setKey] else None if scrapeImages: utils.setImageCopy( self.configuration['exportDir'], self.configuration['images'], image, setKey, dryRun) # Works only if most recent game is first in line (raidendx;raidndx not the opposite) if len(selected) == 0 and not multiGameFoundInSet: notInAnySet.append(game) elif len(selected) == 1: if selected[0] not in onlyInOneSet: onlyInOneSet[selected[0]] = [] onlyInOneSet[selected[0]].append(game) self.logger.log(" " + audit) # writing and closing everything for setKey in self.usingSystems: treeSet = etree.ElementTree(roots[setKey]) treeSet.write(os.path.join(self.configuration['exportDir'], setKey + ".dat"), xml_declaration=True, encoding="utf-8") CSVs[setKey].close() gamelist.closeWrite(gamelists[setKey]) scoreSheet.close() self.logger.log("\n<------------------ RESULTS ------------------>") self.logger.log("NOT FOUND IN ANY SET : " + str(len(notInAnySet)), self.logger.WARNING) self.logger.log(" ".join(notInAnySet), self.logger.WARNING)