def get_code(name: str, include_function: bool) -> Tuple[bool, str, str, str]: # Find the .inc file for the non matching function inc_file = find_inc_file(name) if inc_file is None: return (True, f'No {name}.inc found in asm/non_matching folder.', '', '') src_file = find_source_file(name) if src_file is None: return (True, f'Source file for {name} not found in tmc.map.', '', '') src_file = os.path.join(get_repo_location(), src_file) if not os.path.isfile(src_file): return (True, f'{src_file} is not a file.', '', '') inc_path = inc_file.replace(get_repo_location() + '/', '') (src, signature) = extract_nonmatching_section(inc_path, src_file, include_function) if src is None: return ( True, f'No NONMATCH or ASM_FUNC section found for {inc_path} in {src_file}.', '', '') asm = prepare_asm(inc_file, name) return (False, asm, src, signature)
def get_linker_files() -> List[str]: result = [] with open(os.path.join(settings.get_repo_location(), 'linker.ld'), 'r') as f: for line in f: if '.o' in line: result.append( os.path.join(settings.get_repo_location(), line.split('.o')[0].strip() + '.s')) return result
def split_asm(api: PluginApi, path: str) -> str: filepath = os.path.join(settings.get_repo_location(), 'asm/' + path + '.s') if not os.path.isfile(filepath): api.show_error(name, f'Could not find file: {filepath}') return filename = os.path.split(filepath)[1] foldername = filename[:-2] non_matching_folder = os.path.join('asm', 'non_matching', foldername) non_matching_path = os.path.join(settings.get_repo_location(), non_matching_folder) funcs = parse_file(filepath) text = 'Found functions:\n' for func in funcs: text += f'{func.name}: {len(func.lines)} lines\n' text += f'\nFile: {filepath}\n' text += f'Output folder: {non_matching_path}\n' text += '\nExecute?' if not api.show_question(name, text): return Path(non_matching_path).mkdir(parents=True, exist_ok=True) lines = [] #lines.append('#include "global.h"') #lines.append('#include "entity.h"\n') for func in funcs: funcpath = os.path.join(non_matching_folder, func.name + '.inc') with open(os.path.join(settings.get_repo_location(), funcpath), 'w') as f: f.write('\t.syntax unified\n') f.writelines(func.lines) f.write('\t.syntax divided\n') #print(f'ASM_FUNC("{funcpath}", void {func.name}(Entity* this))\n') lines.append(f'ASM_FUNC("{funcpath}", void {func.name}())\n') #lines.append(f'ASM_FUNC("{funcpath}", void {func.name}(Entity* this))\n') out = '\n'.join(lines) with open(os.path.join(settings.get_repo_location(), 'src', path + '.c'), 'a') as f: f.write(out) api.show_message( name, f'Created file at src/{path}.c. Now change the path in linker.ld.')
def list_all_nonmatch_files() -> List[str]: result = [] for root, dirs, files in os.walk( os.path.join(settings.get_repo_location(), 'src')): for file in files: with open(os.path.join(root, file), 'r') as f: data = f.read() for match in re.findall(r'(?:NONMATCH|ASM_FUNC)\("(.*)"', data): result.append( os.path.join(settings.get_repo_location(), match)) return result
def clang_format(input: str) -> None: ''' Write the code into a temporary file, run clang-format on it and then read the code back. ''' # Format input TMP_FILE = '/tmp/ghidra_code.c' FORMAT_FILE = '/tmp/.clang-format' with open(TMP_FILE, 'w') as f: f.write(input) if not os.path.isfile(FORMAT_FILE): # Need to copy the .clang-format file due to https://stackoverflow.com/a/46374122 subprocess.call([ 'cp', os.path.join(settings.get_repo_location(), '.clang-format'), FORMAT_FILE ]) subprocess.call(['clang-format', '--style=file', '-i', TMP_FILE]) with open(TMP_FILE, 'r') as f: input = f.read() return input
def generate_decomp_me_context(headers: List[str]) -> None: TMP_FILE = '/tmp/test.c' CONTEXT_FILE = 'tmp/decompme-context.c' with open(TMP_FILE, 'w') as output: output.write('#define NULL 0\n') for header in headers: output.write(collect_defines(header)) # Remove empty lines and cc -E comments with open('tmp/test.i', 'r') as file: for line in file: if line.strip() != '' and not line.startswith('#'): output.write(line) # Format with clang-format FORMAT_FILE = '/tmp/.clang-format' if not os.path.isfile(FORMAT_FILE): # Need to copy the .clang-format file due to https://stackoverflow.com/a/46374122 subprocess.call([ 'cp', os.path.join(settings.get_repo_location(), '.clang-format'), FORMAT_FILE ]) subprocess.call(['clang-format', '--style=file', '-i', TMP_FILE]) lines = [] with open(TMP_FILE, 'r') as file: lines = file.readlines() lines = map(lambda x: x.replace(' ', '\t'), lines) with open(CONTEXT_FILE, 'w') as file: file.writelines(lines)
def find_inc_file(name: str) -> Optional[str]: filename = name + '.inc' search_path = os.path.join(get_repo_location(), 'asm', 'non_matching') for root, dirs, files in os.walk(search_path): if filename in files: return os.path.join(root, filename) return None
def export_headers(): INCLUDE_FOLDER = settings.get_repo_location() + '/include' OUTPUT_FOLDER = 'tmp/ghidra_types' # Remove all previous output files for filepath in Path(OUTPUT_FOLDER).rglob('*'): if filepath.is_file(): os.remove(filepath) # Generate new header files for (dirpath, dirnames, filenames) in walk(INCLUDE_FOLDER): for filepath in filenames: rel_dirpath = dirpath[len(INCLUDE_FOLDER) + 1:] abs_path = path.join(dirpath, filepath) rel_path = path.join(rel_dirpath, filepath) lines = open(abs_path, 'r').readlines() Path(path.join(OUTPUT_FOLDER, rel_dirpath)).mkdir(parents=True, exist_ok=True) # Apply general patches for patch in general_patches: lines = patch(lines) # Apply file-specific patches if rel_path in file_specific_patches: lines = file_specific_patches[rel_path](lines) with open(path.join(OUTPUT_FOLDER, rel_path), 'w') as file: file.writelines(lines)
def find_unused(api: PluginApi): all_asm_files = list_all_asm_files() linker_files = get_linker_files() included_files = [ os.path.join(settings.get_repo_location(), x) for x in static_included_files ] nonmatch_files = list_all_nonmatch_files() # Remove linker files unused_asm_files = [ x for x in all_asm_files if not x in linker_files and not x in included_files and not x in nonmatch_files ] if len(unused_asm_files) == 0: api.show_message(name, 'No unused files found.') return text = f'Found {len(unused_asm_files)} unused files:\n' text += '\n'.join(unused_asm_files) text += '\n\nDelete all?' if not api.show_question(name, text): return for file in unused_asm_files: os.remove(file) api.show_message(name, f'Removed {len(unused_asm_files)} unused files.')
def load_symbols(self, rom_variant: RomVariant, silent: bool) -> None: maps = { RomVariant.CUSTOM: 'tmc.map', RomVariant.CUSTOM_EU: 'tmc_eu.map', RomVariant.CUSTOM_JP: 'tmc_jp.map', RomVariant.CUSTOM_DEMO_USA: 'tmc_demo_usa.map', RomVariant.CUSTOM_DEMO_JP: 'tmc_demo_jp.map', } map_file = path.join(settings.get_repo_location(), maps[rom_variant]) if not path.isfile(map_file): if silent: print(f'Could not find tmc.map file at {map_file}.') else: QMessageBox.critical( self, 'Load symbols from .map file', f'Could not find tmc.map file at {map_file}.') return get_symbol_database().load_symbols_from_map(rom_variant, map_file) if not silent: QMessageBox.information( self, 'Load symbols', f'Successfully loaded symbols for {rom_variant} rom from tmc.map file.' )
def list_all_asm_files() -> List[str]: result = [] for root, dirs, files in os.walk( os.path.join(settings.get_repo_location(), 'asm')): for file in files: result.append(os.path.join(root, file)) # TODO would also need to search for .include macros in asm files # Also search unused data asm files for root, dirs, files in os.walk( os.path.join(settings.get_repo_location(), 'data')): for file in files: # TODO maybe check for .inc as well if file.endswith('.s'): result.append(os.path.join(root, file)) return result
def collect_all_headers() -> List[str]: headers = [] include_folder = os.path.join(settings.get_repo_location(), 'include') for root, dirs, files in os.walk(include_folder): for file in files: header_path = os.path.relpath(os.path.join(root, file), include_folder) headers.append(header_path) return headers
def find_source_file(name: str) -> Optional[str]: # Get the source file from tmc.map with open(os.path.join(get_repo_location(), 'tmc.map'), 'r') as f: current_file = None for line in f: if line.startswith(' .text'): current_file = line.split()[3] elif line.strip().endswith(' ' + name): return current_file[0:-2] + '.c' return None
def collect_non_matching_funcs(self): result = [] for root, dirs, files in os.walk(os.path.join(settings.get_repo_location(), 'src')): for file in files: if file.endswith('.c'): with open(os.path.join(root, file), 'r') as f: data = f.read() # Find all NONMATCH and ASM_FUNC macros for match in re.findall(r'(NONMATCH|ASM_FUNC)\(".*",\W*\w*\W*(\w*).*\)', data): result.append(match) return result
def slot_parse_incbins(self) -> None: incbins = [] assembly_extensions = ['.inc', '.s'] for root, dirs, files in os.walk(settings.get_repo_location()): for file in files: filename, file_extension = os.path.splitext(file) if file_extension in assembly_extensions: incbins.extend(self.find_incbins(os.path.join(root, file))) self.incbins = IntervalTree(incbins) self.api.show_message('Pointer Extractor', f'{len(incbins)} .incbins found')
def calc_stats_from_asm_files(self) -> None: asm_folder = os.path.join(settings.get_repo_location(), 'asm', 'non_matching') # Query file sizes asm_files: List[AsmFile] = [] folders: List[FolderStat] = [] for root, dirs, files in os.walk(asm_folder): folder_name = os.path.basename(root) folder_size = 0 for file in files: path = os.path.join(root, file) size = os.path.getsize(path) asm_files.append(AsmFile(file, folder_name, size)) folder_size += size folders.append(FolderStat(folder_name, folder_size)) if True: print('--- Smallest Folders: ---') folders.sort(key=lambda x : x.size) for folder in folders: print('{:<26} {}'.format(folder.name, folder.size)) if True: print('--- Smallest Files ---') asm_files.sort(key=lambda x:x.size) for file in asm_files: print('{:<26} {:<5} {}'.format(file.folder, file.size, file.name[:-4])) if file.size > 250: break; if True: print('--- Stats ---') print(f'{len(asm_files)} functions') if True: foundOB=False foundOBF=False for folder in folders: if folder.name == 'octorokBoss': print(f'OctorokBoss progress: {(1-folder.size/77453)*100:.2f}% ({77453-folder.size}/77453)') foundOB = True if folder.name == 'octorokBossFrozen': print(f'OctorokBossFrozen progress: {(1-folder.size/17832)*100:.2f}% ({17832-folder.size}/17832)') foundOBF = True if foundOB and foundOBF: break
def collect_asm_funcs(self): result = [] src_folder = os.path.join(settings.get_repo_location(), 'src') for root, dirs, files in os.walk(src_folder): for file in files: if file.endswith('.c'): with open(os.path.join(root, file), 'r') as f: data = f.read() # Find all ASM_FUNC macros for match in re.findall( r'ASM_FUNC\(".*",(?: static)?\W*\w*\W*(\w*).*\)', data): result.append( (os.path.relpath(os.path.join(root, file), src_folder), match)) return result
def setup_general_tab(self): self.ui.lineEditUserName.setText(settings.get_username()) self.ui.spinBoxDefaultSelectionSize.setValue( settings.get_default_selection_size()) self.ui.checkBoxAlwaysLoadSymbols.setChecked( settings.is_always_load_symbols()) self.ui.checkBoxHighlight8Bytes.setChecked( settings.is_highlight_8_bytes()) self.ui.spinBoxBytesPerLine.setValue(settings.get_bytes_per_line()) self.ui.checkBoxAutoSave.setChecked(settings.is_auto_save()) self.ui.checkBoxUseConstraints.setChecked( settings.is_using_constraints()) self.ui.lineEditRepoLocation.setText(settings.get_repo_location()) self.ui.toolButtonRepoLocation.clicked.connect(self.edit_repo_location) self.ui.lineEditBuildCommand.setText(settings.get_build_command()) self.ui.lineEditTidyCommand.setText(settings.get_tidy_command())
def generate_struct_definitions() -> None: headers = collect_all_headers() with open('tmp/test.c', 'w') as file: file.write('#define NENT_DEPRECATED\n') for header in headers: file.write(f'#include "{header}"\n') repo_location = settings.get_repo_location() # Preprocess file subprocess.check_call([ 'cc', '-E', '-I', os.path.join(repo_location, 'tools/agbcc'), '-I', os.path.join(repo_location, 'tools/agbcc/include'), '-iquote', os.path.join(repo_location, 'include'), '-nostdinc', '-undef', '-DUSA', '-DREVISION=0', '-DENGLISH', 'tmp/test.c', '-o', 'tmp/test.i' ]) parse_to_json( 'tmp/test.i', get_file_in_database(os.path.join('data_extractor', 'structs.json')))
def find_globals() -> List[TypeDefinition]: globals = [] for (root, dirs, files) in os.walk(os.path.join(get_repo_location(), 'include')): for file in files: defines = {} with open(os.path.join(root, file), 'r') as f: for line in f: match = re.match(r'extern (\w*) (\w*);', line) if match is not None: globals.append( TypeDefinition(match.group(1), match.group(2), '0')) else: match = re.match(r'extern (\w*) (\w*)\[(\w+)\];', line) if match is not None: elements = match.group(3) try: elements = int(elements, 0) except ValueError: if elements in defines: elements = defines[elements] else: raise Exception( f'Unknown array length {elements} for {line}' ) globals.append( TypeDefinition(match.group(1), match.group(2), str(elements))) else: match = re.match(r'#define (\w*) (\w*)', line) if match is not None: value = match.group(2) try: defines[match.group(1)] = int(value, 0) except ValueError: pass return globals
def collect_defines(path: str) -> str: result = '' with open(os.path.join(settings.get_repo_location(), 'include', path), 'r') as file: next_line_belongs_to_define = False first_define = True defined_names = set() if 'isagbprint' in path: defined_names.add('AGBPrintInit()') defined_names.add('AGBPutc(cChr)') defined_names.add('AGBPrint(pBuf)') defined_names.add('AGBPrintf(pBuf,') defined_names.add('AGBPrintFlush1Block()') defined_names.add('AGBPrintFlush()') defined_names.add('AGBAssert(pFile,') for line in file: trimmed_line = line.strip() if next_line_belongs_to_define or trimmed_line.startswith( '#define'): if first_define: first_define = False continue if not next_line_belongs_to_define: # Only use the first define for a name name = trimmed_line.split(' ')[1] if name in defined_names: continue defined_names.add(name) result += line if trimmed_line.endswith('\\'): next_line_belongs_to_define = True else: next_line_belongs_to_define = False return result
def __init__(self, parent, api: PluginApi) -> None: super().__init__('', parent) self.api = api self.ui = Ui_BridgeDock() self.ui.setupUi(self) self.server_thread = None self.observer = None self.modified_timer = None self.slot_server_running(False) self.ui.pushButtonStartServer.clicked.connect(self.slot_start_server) self.ui.pushButtonStopServer.clicked.connect(self.slot_stop_server) self.ui.toolButtonLoadFolder.clicked.connect( self.slot_edit_load_folder) self.ui.toolButtonSaveFolder.clicked.connect( self.slot_edit_save_folder) self.ui.labelConnectionStatus.setText('Server not yet running.') # Initially load from repo folder self.ui.lineEditLoadFolder.setText(settings.get_repo_location()) self.visibilityChanged.connect(self.slot_visibility_changed)
def slot_find_finished(self) -> None: src_folder = os.path.join(settings.get_repo_location(), 'src') finished_files = [] for root, dirs, files in os.walk(src_folder): for file in files: if file.endswith('.c'): abspath = os.path.join(root, file) with open(abspath, 'r') as input: unfinished = 'ASM_FUNC' in input.read() if not unfinished: finished_files.append(os.path.relpath(abspath, src_folder)) print('Finished files:') finished_files.sort() #for file in finished_files: # print(file) with open('tmp/finished_files_new.txt', 'w') as file: file.write('\n'.join(finished_files)) try: check_call(['diff', 'tmp/finished_files.txt', 'tmp/finished_files_new.txt']) print('same') except: print('changed!')
def process(self) -> None: try: print('start') # Load shifted rom rom_original = get_rom(RomVariant.USA) print('Load shifted') rom_path = path.join(settings.get_repo_location(), 'tmc.gba') print(rom_path) if not path.isfile(rom_path): self.signal_fail.emit(f'Shifted rom expected at {rom_path}') return rom_shifted = Rom(rom_path) print('Shifted rom loaded') pointerlist = get_pointer_database().get_pointers(RomVariant.USA) END_OF_USED_DATA = 0xde7da4 errors = [] locations = [] shift_location = 0x108 shift_length = 0x10000 take_long_time = True progress = 0 for i in range(END_OF_USED_DATA): orig = rom_original.get_byte(i) shifted_i = i if i >= shift_location: #print('SHIFT') shifted_i += shift_length #print(i, shifted_i) shifted = rom_shifted.get_byte(shifted_i) if orig != shifted: pointers = pointerlist.get_pointers_at(i) # Test if pointer if len(pointers) > 0: assert shifted == orig + 1 # TODO parse the full pointer continue print(f'{hex(i-2)}\t{orig}\t{shifted}') errors.append((i, orig, shifted)) locations.append(i) #self.signal_fail.emit(f'Failed at {hex(i)}: {orig} {shifted}') #break else: if take_long_time: if rom_original.get_byte(i + 1) != 0x8: # Certainly not a pointer here continue pointers = pointerlist.get_pointers_at(i) if len(pointers) > 0: if pointers[0].address == i - 2: errors.append((i, orig, shifted)) locations.append(i) print(f'missing shift at {hex(i-2)}') #if len(pointers) > 0: # TODO test that pointer was shifted #pass new_progress = i * 100 // END_OF_USED_DATA if new_progress != progress: progress = new_progress self.signal_progress.emit(new_progress) if len(errors) == 0: self.signal_done.emit() else: self.signal_locations.emit(locations) self.signal_fail.emit(f'{len(errors)} errors found.') except Exception as e: print(e) self.signal_fail.emit('Caught exception')
def slot_script_addr(self, addr: int) -> None: print('ADDR: ', addr) if addr == 0: self.ui.labelCode.setText('No script executed in current context.') return rom = get_rom(RomVariant.CUSTOM) # receive the current instruction pointer #instruction_pointer = 0x8009b70 #instruction_pointer = 0x8009d42 instruction_pointer = addr symbols = get_symbol_database().get_symbols( RomVariant.CUSTOM) # Symbols for our custom USA rom symbol = symbols.get_symbol_at(instruction_pointer - ROM_OFFSET) script_name = symbol.name script_offset = instruction_pointer - ROM_OFFSET - symbol.address # Find file containing the script # TODO or statically find all script files? script_file = None for root, dirs, files in os.walk( os.path.join(settings.get_repo_location(), 'data', 'scripts')): if script_name + '.inc' in files: script_file = os.path.join(root, script_name + '.inc') break # TODO search the file contents for the script for file in files: path = os.path.join(root, file) with open(path, 'r') as f: if 'SCRIPT_START ' + script_name in f.read(): script_file = path if script_file is None: self.ui.labelCode.setText( f'ERROR: Count not find script file containing {script_name}') return self.ui.labelScriptName.setText(script_file) script_lines = [] with open(script_file, 'r') as file: script_lines = file.read().split('\n') # print(script_lines) # TODO for testing ifdefs: script_0800B200 # print('test') # print(symbol) # print(script_offset) # TODO only disassemble the number of bytes, the actual instructions are not interesting as they are read from the source file. (_, instructions) = disassemble_script( rom.get_bytes(symbol.address, symbol.address + symbol.length), symbol.address) output = '' current_instruction = 0 in_correct_script = False ifdef_stack = [True] for line in script_lines: stripped = line.strip() if stripped.startswith('SCRIPT_START'): in_correct_script = stripped == 'SCRIPT_START ' + script_name output += f'{line}\n' continue if not in_correct_script or stripped.startswith( '@') or stripped.endswith(':'): output += f'{line}\n' continue if '.ifdef' in stripped: if not ifdef_stack[-1]: ifdef_stack.append(False) output += f'{line}\n' continue # TODO check variant is_usa = stripped.split(' ')[1] == 'USA' ifdef_stack.append(is_usa) output += f'{line}\n' continue if '.ifndef' in stripped: if not ifdef_stack[-1]: ifdef_stack.append(False) output += f'{line}\n' continue is_usa = stripped.split(' ')[1] == 'USA' ifdef_stack.append(not is_usa) output += f'{line}\n' continue if '.else' in stripped: if ifdef_stack[-2]: # If the outermost ifdef is not true, this else does not change the validiness of this ifdef ifdef_stack[-1] = not ifdef_stack[-1] output += f'{line}\n' continue if '.endif' in stripped: ifdef_stack.pop() output += f'{line}\n' continue if not ifdef_stack[-1]: # Not defined for this variant output += f'{line}\n' continue if current_instruction >= len(instructions): # TODO maybe even not print additional lines? output += f'{line}\n' continue addr = instructions[current_instruction].addr prefix = '' if addr == script_offset: prefix = '>' output += f'{addr:03d}| {prefix}{line}\t\n' current_instruction += 1 if stripped.startswith('SCRIPT_END'): break self.ui.labelCode.setText(output)
def get_all_asset_configs() -> List[str]: return [ x for x in os.listdir( os.path.join(settings.get_repo_location(), 'assets')) if x.endswith('.json') ]
def read_assets(name: str) -> Assets: with open(os.path.join(settings.get_repo_location(), 'assets', name), 'r') as file: return Assets(json.load(file))
def write_assets(name: str, assets: Assets) -> None: with open(os.path.join(settings.get_repo_location(), 'assets', name), 'w') as file: json.dump(assets.assets, file, indent=2)
def export_incbins(api: PluginApi) -> None: for (root, dirs, files) in os.walk(os.path.join(get_repo_location(), 'data')): for file in files: filepath = os.path.join(root, file) parse_file(filepath)
def store_code(name: str, includes: str, header: str, src: str, matching: bool) -> Tuple[bool, str]: # Find the .inc file for the non matching function inc_file = find_inc_file(name) if inc_file is None: return (True, f'No {name}.inc found in asm/non_matching folder.') src_file = find_source_file(name) if src_file is None: return (True, f'Source file for {name} not found in tmc.map.') src_file = os.path.join(get_repo_location(), src_file) if not os.path.isfile(src_file): return (True, f'{src_file} is not a file.') inc_path = inc_file.replace(get_repo_location() + '/', '') (headers, data) = read_file_split_headers(src_file) # https://stackoverflow.com/a/23146126 def find_last_containing(lst, sought_elt): for r_idx, elt in enumerate(reversed(lst)): if sought_elt in elt: return len(lst) - 1 - r_idx # Insert includes at the correct place if includes.strip() != '': last_include_index = find_last_containing(headers, '#include') headers.insert(last_include_index + 1, includes.strip() + '\n') # Append headers if header.strip() != '': headers.append(header.strip() + '\n\n') # Add NONMATCH macro to replacement string when not matching if not matching: src = re.sub(r'(.*?)\s*{', r'NONMATCH("' + inc_path + r'", \1) {', src, 1) + '\nEND_NONMATCH' match = re.search( r'NONMATCH\(\"' + re.escape(inc_path) + r'\", ?(.*?)\) ?{(.*?)END_NONMATCH', ''.join(data), re.MULTILINE | re.DOTALL) if match: data = re.sub(r'NONMATCH\(\"' + re.escape(inc_path) + r'\", ?(.*?)\) ?{(.*?)END_NONMATCH', src, ''.join(data), flags=re.MULTILINE | re.DOTALL) else: match = re.search( r'ASM_FUNC\(\"' + re.escape(inc_path) + r'\", ?(.*?)\)$', ''.join(data), re.MULTILINE | re.DOTALL) if match: data = re.sub(r'ASM_FUNC\(\"' + re.escape(inc_path) + r'\", ?(.*?)\)$', src, ''.join(data), flags=re.MULTILINE | re.DOTALL) else: return ( True, f'No NONMATCH or ASM_FUNC section found for {inc_path} in {src_file}.' ) with open(src_file, 'w') as f: f.write(''.join(headers)) f.write(data) if matching: # Remove the .inc file as its no longer neede os.remove(inc_file) return (False, '')