def build_clone_lists(input_dat):
""" Formats a clone list appriopriately """
# Import JSON files that have the same name as dat_name + .json
remove_string = ' \((Parent-Clone|J64|ROM|Decrypted|Encrypted|BigEndian|ByteSwapped)\)'
if re.search(remove_string, input_dat.name) != None:
dat_name = re.sub(remove_string, '', input_dat.name)
else:
dat_name = input_dat.name
if 'GameCube' in dat_name and ('NKit GCZ' in dat_name or 'NKit ISO'
in dat_name or 'NASOS' in dat_name):
clone_file = './clonelists/Nintendo - GameCube.json'
elif 'Wii U' in dat_name and 'WUX' in dat_name:
clone_file = './clonelists/Nintendo - Wii U.json'
elif 'Wii' in dat_name and ('NKit GCZ' in dat_name or 'NKit ISO'
in dat_name or 'NASOs' in dat_name):
clone_file = './clonelists/Nintendo - Wii.json'
elif ('PlayStation Portable' in dat_name and '(PSN)' not in dat_name
and '(PSX2PSP)' not in dat_name and '(UMD Music)' not in dat_name
and '(UMD Video)' not in dat_name):
if 'no-intro' in input_dat.url:
clone_file = './clonelists/Sony - PlayStation Portable (No-Intro).json'
elif 'redump' in input_dat.url:
clone_file = './clonelists/Sony - PlayStation Portable (Redump).json'
else:
clone_file = './clonelists/' + dat_name + '.json'
if os.path.exists(clone_file) == True and os.path.isfile(
clone_file) == True:
try:
with open(clone_file, 'r') as input_file_read:
clonedata = json.load(input_file_read)
except OSError as e:
print(f'\n{Font.error_bold}* Error: {Font.end}{str(e)}\n')
raise
except ValueError as e:
printwrap(
f'\n{Font.error_bold}* Error: "{os.path.abspath(clone_file)}"{Font.error} isn\'t valid JSON. Exiting...{Font.end}',
'error')
print('\n')
raise
compilations = set()
conditional_overrides = {}
overrides = {}
renames = {}
if 'compilations' in clonedata:
compilations.update(clonedata['compilations'])
if 'overrides' in clonedata:
overrides = clonedata['overrides']
if 'conditional_overrides' in clonedata:
conditional_overrides = clonedata['conditional_overrides']
if 'renames' in clonedata:
renames = clonedata['renames']
return CloneList(compilations, overrides, conditional_overrides,
renames)
def build_regions(REGIONS):
""" Imports regions and languages from a file """
regions = Regions()
if (os.path.exists(REGIONS.filename) == True
and os.path.isfile(REGIONS.filename) == True):
try:
with open(REGIONS.filename, 'r') as input_file_read:
regiondata = json.load(input_file_read)
# Set the implied languages
if REGIONS.region_order in regiondata:
for region, language in regiondata[
REGIONS.region_order].items():
regions.all.append(region)
regions.implied_language = regiondata[REGIONS.region_order]
# Set the default region order
regions.region_order = [
region for region in regiondata[REGIONS.region_order]
]
else:
printwrap(
f'{Font.error_bold}* Error: {Font.error}The {Font.bold}'
f'{REGIONS.region_order}{Font.error} key is missing '
f'from {Font.bold}{REGIONS.filename}{Font.error}. It\'s needed '
'for Retool to know what regions and languages are available.'
f'{Font.end}', 'error')
sys.exit()
# Set the other language details
if REGIONS.languages in regiondata:
regions.languages_key = regiondata[REGIONS.languages]
regions.languages_long = [
language for language in regiondata[REGIONS.languages]
]
regions.languages_short = []
for long_language, short_language in regiondata[
REGIONS.languages].items():
regions.languages_short.append(short_language)
input_file_read.close()
return regions
except OSError as e:
print(f'\n{Font.error_bold}* Error: {Font.end}{str(e)}\n')
raise
else:
printwrap(
f'{Font.error_bold}* Error: {Font.error}The {Font.bold}'
f'{REGIONS.filename}{Font.error} file is missing. Retool can\'t continue.'
f'{Font.end}', 'error')
sys.exit()
def build_tags(TAGS):
""" Imports a list of tags from a file, to strip from titles during processing """
tag_strings = Tags()
if (os.path.exists(TAGS.filename) == True
and os.path.isfile(TAGS.filename) == True):
try:
with open(TAGS.filename, 'r') as input_file_read:
tag_file = json.load(input_file_read)
if TAGS.ignore in tag_file:
tag_strings.ignore = tag_file[TAGS.ignore]
for tag in tag_file[TAGS.demote_editions] + tag_file[
TAGS.promote_editions]:
if tag not in tag_file[TAGS.ignore]:
tag_strings.ignore.append(tag)
if TAGS.disc_rename in tag_file:
tag_strings.disc_rename = tag_file[TAGS.disc_rename]
if TAGS.promote_editions in tag_file:
tag_strings.promote_editions = tag_file[
TAGS.promote_editions]
if TAGS.demote_editions in tag_file:
tag_strings.demote_editions = tag_file[
TAGS.demote_editions]
# Error handling
for section in [
TAGS.ignore, TAGS.disc_rename, TAGS.promote_editions,
TAGS.demote_editions
]:
if section not in tag_file:
printwrap(
f'{Font.warning}* The {Font.bold}{section}{Font.warning}'
f' key is missing from {Font.bold}{TAGS.filename}'
f'{Font.warning}. Clone matching won\'t be accurate.'
f'{Font.end}')
input_file_read.close()
return tag_strings
except OSError as e:
print(f'\n{Font.error_bold}* Error: {Font.end}{str(e)}\n')
raise
else:
printwrap(
f'{Font.error_bold}* Error:{Font.error} The {Font.bold}{TAGS.filename}'
f'{Font.error} file is missing.{Font.end}')
sys.exit()
def build_clone_lists(dat_name):
""" Formats a clone list appriopriately """
# Import JSON files that have the same name as dat_name + .json
if 'GameCube' in dat_name and ('NKit GCZ' in dat_name or 'NKit ISO'
in dat_name or 'NASOS' in dat_name):
clone_file = './clonelists/Nintendo - GameCube.json'
elif 'Wii U' in dat_name and 'WUX' in dat_name:
clone_file = './clonelists/Nintendo - Wii U.json'
elif 'Wii' in dat_name and ('NKit GCZ' in dat_name or 'NKit ISO'
in dat_name or 'NASOs' in dat_name):
clone_file = './clonelists/Nintendo - Wii.json'
else:
clone_file = './clonelists/' + dat_name + '.json'
if os.path.exists(clone_file) == True and os.path.isfile(
clone_file) == True:
try:
with open(clone_file, 'r') as input_file_read:
clonedata = json.load(input_file_read)
except OSError as e:
print(f'\n{Font.error_bold}* Error: {Font.end}{str(e)}\n')
raise
except ValueError as e:
printwrap(
f'\n{Font.error_bold}* Error: "{os.path.abspath(clone_file)}"{Font.error} isn\'t valid JSON. Exiting...{Font.end}',
'error')
print('\n')
raise
compilations = set()
conditional_overrides = {}
overrides = {}
renames = {}
if 'compilations' in clonedata:
compilations.update(clonedata['compilations'])
if 'overrides' in clonedata:
overrides = clonedata['overrides']
if 'conditional_overrides' in clonedata:
conditional_overrides = clonedata['conditional_overrides']
if 'renames' in clonedata:
renames = clonedata['renames']
return CloneList(compilations, overrides, conditional_overrides,
renames)
def generate_config(region_data):
if not os.path.isfile('user-config.yaml'):
try:
with open('user-config.yaml', 'w') as output_file:
output_file.writelines(
'---\n# If the -l option is used, only include titles with the following languages.'
)
output_file.writelines(
'\n# Comment out languages you don\'t want.')
output_file.writelines('\n- language filter:')
def write_entry(string, comment=False):
if comment == True:
output_file.writelines(f'\n # - {string}')
else:
output_file.writelines(f'\n - {string}')
output_file.writelines(f'\n - English')
for language in region_data.languages_long:
if language != 'English':
write_entry(language, True)
output_file.writelines(
'\n\n# The region order Retool follows. Comment out the regions you don\'t want.'
)
output_file.writelines('\n- region order:')
for region in region_data.region_order:
write_entry(region)
printwrap(
f'{Font.warning}* The {Font.warning_bold}user-config.yaml '
f'{Font.warning}file was missing, so a new one has been generated. '
'You might want to edit it to define a custom region order, or to '
f'filter specific languages. You can now run Retool '
f'normally.{Font.end}', 'error')
sys.exit()
except OSError as e:
print(f'\n{Font.error_bold}* Error: {Font.end}{str(e)}\n')
raise
def main():
# Start a timer from when the process started
start_time = time.time()
# Splash screen
os.system('cls' if os.name == 'nt' else 'clear')
print(f'{Font.bold}\nRetool {__version__}{Font.end}')
print('-----------')
if len(sys.argv) == 1:
printwrap(
f'Creates 1G1R versions of Redump ({Font.underline}'
f'http://redump.org/{Font.end}) dats.', 'no_indent'
)
# Generate regions and languages
region_data = build_regions(RegionKeys())
LANGUAGES = '|'.join(region_data.languages_short)
# Regexes
REGEX = Regex(LANGUAGES)
# Generate user config file if it's missing
generate_config(region_data)
# Check user input -- if none, or there's an error, available options will be shown
user_input = check_input()
# Import the user-config.yaml file and assign filtered languages and custom
# region order.
user_input = import_user_config(region_data, user_input)
# Based on region counts from redump.org
PRIORITY_REGIONS = [
'USA', 'Japan', 'Europe', 'Germany', 'Poland', 'Italy',
'France', 'Spain', 'Netherlands', 'Russia', 'Korea']
# Generate tag strings
user_input.tag_strings = build_tags(TagKeys())
if os.path.isfile('.dev'):
printverbose(
user_input.verbose,
f'{Font.warning_bold}* Operating in dev mode{Font.end}')
# Process the input file or folder
if os.path.isdir(user_input.input_file_name) == True:
is_folder = True
dat_files = glob.glob(os.path.abspath(user_input.input_file_name) + '/*.dat')
print('Processing folder...')
else:
is_folder = False
dat_files = {user_input.input_file_name}
file_count = len(dat_files)
for i, dat_file in enumerate(dat_files):
if file_count > 1:
print(f'\n{Font.underline}Processing file '
f'{i+1}/{len(dat_files)}{Font.end}\n')
# Process and get the details we need from the input file
input_dat = process_input_dat(dat_file, is_folder)
# Import the system's clone lists, if they exist
input_dat.clone_lists = build_clone_lists(input_dat.name)
# Import scraped Redump metadata for titles
input_dat.metadata = import_metadata(input_dat.name)
# Get the stats from the original soup object before it's changed later
print('* Gathering stats... ', sep=' ', end='', flush=True)
stats = Stats(len(input_dat.soup.find_all('game')), user_input, input_dat)
print('done.')
# Provide dat details to reassure the user the correct file is being processed
print(f'\n| {Font.bold}DAT DETAILS{Font.end}')
print(f'| Description: {input_dat.description}')
print(f'| Author: {input_dat.author}')
print(f'| URL: {input_dat.url}')
print(f'| Version: {input_dat.version}\n')
# For performance, change the region order so titles with a lot of regions are
# processed first, and unknown regions are processed last. This doesn't affect
# the user's region order when it comes to title selection.
processing_region_order = [
x for x in user_input.user_region_order if x in PRIORITY_REGIONS]
processing_region_order.extend(
[x for x in user_input.user_region_order if x not in PRIORITY_REGIONS and x != 'Unknown'])
if 'Unknown' in user_input.user_region_order:
processing_region_order.append('Unknown')
# Convert each region's XML to dicts so we can more easily work with the data,
# and determine each region's parent
titles = Titles()
compilations_found = set()
for region in processing_region_order:
print(
f'* Checking dat for titles in provided regions... {region}',
sep='', end='\r', flush=True
)
titles.regions[region] = dat_to_dict(
region, region_data, input_dat, user_input,
compilations_found, REGEX)
sys.stdout.write("\033[K")
# Deal with compilations
if input_dat.clone_lists != None:
if user_input.no_compilations == True:
missing_compilations = {
compilation for compilation in input_dat.clone_lists.compilations if compilation not in compilations_found}
for compilation in missing_compilations:
printverbose(
user_input.verbose,
f'{Font.warning_bold}* Title in compilations list not found in dat or selected regions: '
f'{compilation}{Font.end}')
stats.compilations_count = len(compilations_found)
print('* Checking dat for titles in provided regions... done.')
# Combine all regions' titles and choose a parent based on region order
print('* Finding parents across regions... ', sep='', end='\r', flush=True)
titles = choose_cross_region_parents(titles, user_input)
print('* Finding parents across regions... done.')
# Process clone lists
if input_dat.clone_lists != None:
print('* Assigning clones from clone lists... ', sep='', end='\r', flush=True)
titles = assign_clones(titles, input_dat, region_data, user_input, REGEX)
sys.stdout.write("\033[K")
print('* Assigning clones from clone lists... done.')
# Get the clone count
stats.clone_count = 0
for group, disc_titles in titles.all.items():
for disc_title in disc_titles:
if disc_title.cloneof != '':
stats.clone_count += 1
# Get final title count
if user_input.legacy == False:
stats.final_title_count = get_title_count(titles, is_folder) - stats.clone_count
else:
stats.final_title_count = get_title_count(titles, is_folder)
# Name the output file
output_file_name = (
os.path.join(
user_input.output_folder_name,
f'{input_dat.name} ({str("{:,}".format(stats.final_title_count))}) ({input_dat.version}) '
f'[1G1R]{user_input.user_options} (Retool {datetime.datetime.now().strftime("%Y-%m-%d %H-%M-%SS")[:-1]}).dat'))
# Write the output dat file
write_dat_file(input_dat, user_input, output_file_name, stats, titles)
# Report stats
report_stats(stats, titles, user_input, input_dat, region_data)
# Start the loop again if processing a folder
if stats.final_title_count == 0 and is_folder == True: return
# Stop the timer
stop_time = time.time()
total_time_elapsed = str('{0:.2f}'.format(round(stop_time - start_time,2)))
# Set the summary message if input was a folder and files were found
if is_folder == True:
if file_count > 0:
if file_count == 1:
file_noun = 'file'
else:
file_noun = 'files'
file_count = str('{:,}'.format(file_count))
finish_message = (
f'{Font.success}* Finished processing {file_count} {file_noun} in the '
f'{Font.bold}"{user_input.input_file_name}{Font.success}" folder in '
f'{total_time_elapsed}s. 1G1R dats have been created in the '
f'{Font.bold}"{user_input.output_folder_name}"{Font.success} folder.{Font.end}'
)
else:
# Set the summary message if no files were found in input folder
finish_message = (
f'{Font.warning}* No files found to process in the '
f'{Font.bold}"{user_input.input_file_name}"{Font.warning} folder.{Font.end}'
)
else:
# Set the summary message if input was a single file
finish_message = (
f'{Font.success}* Finished adding '
f'{str("{:,}".format(stats.final_title_count))}'
f' unique titles to "{Font.bold}{output_file_name}" '
f'{Font.success}in {total_time_elapsed}s.{Font.end}'
)
# Print the summary message
print('\n')
printwrap(f'{finish_message}\n')
return
def process_input_dat(dat_file, is_folder):
""" Prepares input dat file and converts to an object
Returns a Dat object with the following populated:
.name
.description
.version
.author
.url
.soup
Removes the following from a Dat object:
.contents
"""
if is_folder == True:
next_status = ' Skipping file...'
else:
next_status = ''
printwrap(f'* Reading dat file: "{Font.bold}{dat_file}{Font.end}"')
try:
with open(dat_file, 'r') as input_file:
print('* Validating dat file... ', sep=' ', end='', flush=True)
input_dat = input_file.read()
except OSError as e:
printwrap(
f'{Font.error_bold}* Error: {Font.error}{str(e)}.{Font.end}{next_status}',
'error')
if is_folder == False:
raise
else:
return
# Check the dat file format -- if it's CLRMAMEPro format, convert it to LogiqX
clrmame_header = re.search('^clrmamepro \($.*?^\)$', input_dat,
re.M | re.S)
if clrmame_header:
print('file is a CLRMAMEPro dat file.')
input_dat = convert_clrmame_dat(clrmame_header, input_dat, is_folder)
# Go to the next file in a batch operation if something went wrong.
if input_dat == 'end_batch': return
else:
input_dat = Dat(input_dat)
# Exit if there are entity or element tags to avoid abuse
if '<!ENTITY' in input_dat.contents or '<!ELEMENT' in input_dat.contents:
print('failed.')
printwrap(
f'{Font.error_bold} Error: {Font.error}Entity and element tags '
f'aren\'t supported in dat files.{Font.end}{next_status}',
'error')
sys.exit()
# Check for a valid Redump XML dat that follows the Logiqx dtd
valid_dat_file = False
if ('<datafile>' in input_dat.contents
and '<?xml' in input_dat.contents
and '<game' in input_dat.contents):
# Remove unexpected XML declarations from the file so we can check validity
try:
input_dat.contents = input_dat.contents.replace(
re.search('<\?xml.*?>', input_dat.contents)[0],
'<?xml version="1.0"?>')
except:
print('failed.')
printwrap(
f'{Font.error_bold}* Error: {Font.error}File is missing an XML '
f'declaration. It\'s probably not a dat file.'
f'{next_status}{Font.end}', 'error')
if is_folder == False:
sys.exit()
else:
return
try:
with open('datafile.dtd') as dtdfile:
dtd = etree.DTD(dtdfile)
try:
root = etree.XML(input_dat.contents)
if dtd.validate(root) == False:
print('failed.')
printwrap(
f'{Font.error_bold}* Error: {Font.error}XML file'
f'doesn\'t conform to Logiqx dtd. '
f'{dtd.error_log.last_error}.'
f'{next_status}{Font.end}', 'error')
if is_folder == False:
sys.exit()
else:
return
except etree.XMLSyntaxError as e:
print('failed.')
printwrap(
f'{Font.error_bold}* Error: {Font.error}XML file is '
f'malformed. {e}.{next_status}{Font.end}', 'error')
if is_folder == False:
sys.exit()
else:
return
else:
print('file is a Logiqx dat file.')
except OSError as e:
printwrap(
f'{Font.error_bold}* Error: {str(e)}{next_status}{Font.end}',
'error')
if is_folder == False:
raise
else:
return
else:
print('failed.')
printwrap(
f'{Font.error_bold}* Error: "{dat_file}"{Font.error} '
f'isn\'t a compatible dat file.{next_status}{Font.end}',
'error')
if is_folder == False:
sys.exit()
else:
return
# Convert contents to BeautifulSoup object, remove original contents attribute
print('* Converting dat file to a searchable format... ',
sep=' ',
end='',
flush=True)
input_dat.soup = BeautifulSoup(input_dat.contents, "lxml-xml")
del input_dat.contents
print('done.')
# Set input dat header details
if input_dat.soup.find('header') != None:
for key, value in input_dat.__dict__.items():
if (key != 'soup' and key != 'user_options' and value == 'Unknown'
and input_dat.soup.find(key) != None):
setattr(input_dat, key, input_dat.soup.find(key).string)
elif value == '':
setattr(input_dat, key, 'Unknown')
# Remove Retool tag from name if it exists
input_dat.name = input_dat.name.replace(' (Retool)', '')
# Sanitize some header details which are used in the output filename
characters = [':', '\\', '/', '<', '>', '"', '|', '?', '*']
reserved_filenames = ['con', 'prn', 'aux', 'nul', 'com[1-9]', 'lpt[1-9]']
for character in characters:
if character in input_dat.name:
input_dat.name = input_dat.name.replace(character, '-')
if character in input_dat.version:
input_dat.version = input_dat.version.replace(character, '-')
for filename in reserved_filenames:
if re.search('^' + filename + '$', input_dat.name) != None:
input_dat.name = 'Unknown'
if re.search('^' + filename + '$', input_dat.version) != None:
input_dat.version = 'Unknown'
return input_dat
def convert_clrmame_dat(clrmame_header, input_dat, is_folder):
""" Converts CLRMAMEPro dat format to LogiqX dat format """
def header_details(find_string, replace_string):
""" Gets values for CLRMAMEPro dat header details """
search_string = re.search(find_string, clrmame_header[0])
if search_string != None:
return re.sub(replace_string, '', search_string.group(0)).strip()
else:
return ''
dat_name = header_details(re.compile('.*?name.*'), 'name |(\")')
dat_description = header_details(re.compile('.*?description.*'),
'description |(\")')
dat_category = header_details(re.compile('.*?category.*'),
'category |(\")')
dat_version = header_details(re.compile('.*?version.*'), 'version |(\")')
dat_author = header_details(re.compile('.*?author.*'), 'author |(\")')
convert_dat = []
convert_dat.append('<?xml version="1.0"?>\n\
<!DOCTYPE datafile PUBLIC "-//Logiqx//DTD ROM Management Datafile//EN" \
"http://www.logiqx.com/Dats/datafile.dtd"><datafile>\n\t<header>')
convert_dat.append(f'\t\t<name>{dat_name}</name>')
convert_dat.append(f'\t\t<description>{dat_description}</description>')
convert_dat.append(f'\t\t<version>{dat_version}</version>')
convert_dat.append(f'\t\t<author>{dat_author}</author>\n\t</header>')
# Now work through each of the title details
dat_contents = re.findall('^game \($.*?^\)$', input_dat, re.M | re.S)
if dat_contents:
for item in dat_contents:
xml_node = re.split('\n', item)
regex = re.sub('name |(\")', '', xml_node[1].strip())
convert_dat.append(
f'\t<game name="{regex}">'
f'\n\t\t<category>{dat_category}</category>\n\t\t<description>'
f'{regex}</description>')
for node in xml_node:
if node.strip().startswith('rom'):
node = node
node = re.sub('^rom \( name ', '<rom name="', node.strip())
node = re.sub(' size ', '" size="', node.strip())
node = re.sub(' crc ', '" crc="', node.strip())
node = re.sub(' md5 ', '" md5="', node.strip())
node = re.sub(' sha1 ', '" sha1="', node.strip())
node = re.sub(' \)$', '" />', node.strip())
convert_dat.append('\t\t' + node)
convert_dat.append('\t</game>')
convert_dat.append('</datafile>')
convert_dat = '\n'.join(convert_dat)
else:
printwrap(
f'{Font.error_bold} * Error: {Font.error}file isn\'t Logiqx XML or '
f'CLRMAMEPro dat.{Font.end}', 'error')
if is_folder == False:
sys.exit()
else:
return 'end_batch'
return Dat(convert_dat, dat_name, dat_description, dat_version, dat_author)
