def validate_dat(file: Path, use_hashes: bool) -> None: root = datafile.parse(file, silence=True) has_cloneof = False lacks_sha1 = False offending_entry = '' for game in root.game: if game.cloneof: has_cloneof = True break for game in root.game: for game_rom in game.rom: if not game_rom.sha1: lacks_sha1 = True offending_entry = game.name break if use_hashes and lacks_sha1: sys.exit( 'ERROR: Cannot use hash information because DAT lacks SHA1 digests ' 'for [%s].' % offending_entry) if not has_cloneof: print('This DAT *seems* to be a Standard DAT', file=sys.stderr) print( 'A Parent/Clone XML DAT is required to generate a 1G1R ROM set', file=sys.stderr) if use_hashes: print( 'If you are using this to rename files based on their hashes, ' 'a Standard DAT is enough', file=sys.stderr) print('Do you want to continue anyway? (y/n)', file=sys.stderr) answer = input() if answer.strip() not in ('y', 'Y'): sys.exit()
def index_files( input_dir: Path, dat_file: Path) -> Dict[str, Optional[Path]]: result: Dict[str, Optional[Path]] = {} also_check_archive: bool = False root = datafile.parse(dat_file, silence=True) global RULES if not RULES: RULES = get_header_rules(root) for game in root.game: for rom_entry in game.rom: result[rom_entry.sha1.lower()] = None also_check_archive |= bool(ZIP_REGEX.search(rom_entry.name)) print('Scanning directory: %s\033[K' % input_dir, file=sys.stderr) files_data = [] for full_path in input_dir.rglob('*'): if not full_path.is_file(): continue try: print( '%s%s\033[K' % ( FOUND_PREFIX, trim_to( full_path.relative_to(input_dir), available_columns(FOUND_PREFIX) - 2)), end='\r', file=sys.stderr) file_size = full_path.stat().st_size files_data.append(FileData(file_size, full_path)) except OSError as e: print( 'Error while reading file: %s\033[K' % e, file=sys.stderr) files_data.sort(key=FileData.get_size, reverse=True) print('%s%i files\033[K' % (FOUND_PREFIX, len(files_data)), file=sys.stderr) if files_data: global PROGRESSBAR PROGRESSBAR = MultiThreadedProgressBar( len(files_data), THREADS, prefix='Calculating hashes') PROGRESSBAR.init() def process_thread_with_progress( shared_files_data: List[FileData], shared_result_data: List[Dict[str, Path]]) -> None: curr_thread = current_thread() if not isinstance(curr_thread, IndexedThread): sys.exit('Bad thread type. Expected %s' % IndexedThread) while True: try: next_file = shared_files_data.pop(0) PROGRESSBAR.print_thread( curr_thread.index, next_file.path.relative_to(input_dir)) shared_result_data.append(process_file( next_file, also_check_archive)) PROGRESSBAR.print_bar() except IndexError: PROGRESSBAR.print_thread(curr_thread.index, "DONE") break threads = [] intermediate_results = [] for i in range(0, THREADS): t = IndexedThread( index=i, target=process_thread_with_progress, args=[files_data, intermediate_results], daemon=True) t.start() threads.append(t) for t in threads: t.join() print('\n', file=sys.stderr) for intermediate_result in intermediate_results: for key, value in intermediate_result.items(): if key in result and not \ (result[key] and is_zipfile(result[key])): result[key] = value return result
def parse_games( file: Path, filter_bios: bool, filter_program: bool, filter_enhancement_chip: bool, filter_pirate: bool, filter_promo: bool, filter_unlicensed: bool, filter_proto: bool, filter_beta: bool, filter_demo: bool, filter_sample: bool, exclude: List[Pattern]) -> Dict[str, List[GameEntry]]: games = {} root = datafile.parse(file, silence=True) for input_index in range(0, len(root.game)): game = root.game[input_index] beta_match = BETA_REGEX.search(game.name) demo_match = DEMO_REGEX.search(game.name) sample_match = SAMPLE_REGEX.search(game.name) proto_match = PROTO_REGEX.search(game.name) if filter_bios and BIOS_REGEX.search(game.name): continue if filter_unlicensed and UNL_REGEX.search(game.name): continue if filter_pirate and PIRATE_REGEX.search(game.name): continue if filter_promo and PROMO_REGEX.search(game.name): continue if filter_program and PROGRAM_REGEX.search(game.name): continue if filter_enhancement_chip and ENHANCEMENT_CHIP_REGEX.search(game.name): continue if filter_beta and beta_match: continue if filter_demo and demo_match: continue if filter_sample and sample_match: continue if filter_proto and proto_match: continue if check_in_pattern_list(game.name, exclude): continue is_parent = not game.cloneof is_bad = bool(BAD_REGEX.search(game.name)) beta = parse_prerelease(beta_match) demo = parse_prerelease(demo_match) sample = parse_prerelease(sample_match) proto = parse_prerelease(proto_match) is_prerelease = bool( beta_match or demo_match or sample_match or proto_match) revision = parse_revision(game.name) version = parse_version(game.name) region_data = parse_region_data(game.name) for release in game.release: if release.region and not is_present(release.region, region_data): region_data.append(get_region_data(release.region)) languages = parse_languages(game.name) if not languages: languages = get_languages(region_data) parent_name = game.cloneof if game.cloneof else game.name region_codes = [rd.code for rd in region_data] game_entries: List[GameEntry] = [] for region in region_codes: game_entries.append( GameEntry( is_bad, is_prerelease, region, languages, input_index, revision, version, sample, demo, beta, proto, is_parent, game.name, game.rom if game.rom else [])) if game_entries: if parent_name not in games: games[parent_name] = game_entries else: games[parent_name].extend(game_entries) else: log('WARNING [%s]: no recognizable regions found' % game.name) if not game.rom: log('WARNING [%s]: no ROMs found in the DAT file' % game.name) return games