def load_symbols(self, rom_variant: RomVariant, silent: bool) -> None:
maps = {
RomVariant.CUSTOM: 'tmc.map',
RomVariant.CUSTOM_EU: 'tmc_eu.map',
RomVariant.CUSTOM_JP: 'tmc_jp.map',
RomVariant.CUSTOM_DEMO_USA: 'tmc_demo_usa.map',
RomVariant.CUSTOM_DEMO_JP: 'tmc_demo_jp.map',
}
map_file = path.join(settings.get_repo_location(), maps[rom_variant])
if not path.isfile(map_file):
if silent:
print(f'Could not find tmc.map file at {map_file}.')
else:
QMessageBox.critical(
self, 'Load symbols from .map file',
f'Could not find tmc.map file at {map_file}.')
return
get_symbol_database().load_symbols_from_map(rom_variant, map_file)
if not silent:
QMessageBox.information(
self, 'Load symbols',
f'Successfully loaded symbols for {rom_variant} rom from tmc.map file.'
)
def slot_find_nonmatching(self) -> None:
symbols = get_symbol_database().get_symbols(
RomVariant.CUSTOM) # Symbols for our custom USA rom
nonmatch = self.collect_non_matching_funcs()
with open('tmp/nonmatching.html', 'w') as out:
out.write(
'<link rel="stylesheet" href="theme.css" /><script src="sortable.min.js"></script><table class="sortable-theme-slick" data-sortable><thead><tr><th>File</th><th>Function</th><th>Size</th><th data-sortable="false"></th></thead><tbody>\n'
)
for (file, func) in nonmatch:
symbol = symbols.find_symbol_by_name(func)
size = 0
if symbol is None:
print(f'No symbol found for {func}, maybe static?')
return
else:
size = symbol.length
(err, asm, src, signature) = get_code(func, True)
if err:
self.api.show_error(self.name, asm)
return
url = generate_cexplore_url(src, asm)
out.write(
f'<tr><td>{file}</td><td>{func}</td><td>{size}</td><td><a href="{url}">CExplore</a></td></tr>\n'
)
out.write('</tbody></table>\n')
self.api.show_message(self.name, 'Wrote to tmp/nonmatching.html')
def slot_show_bridge(self) -> None:
if get_symbol_database().get_symbols(RomVariant.CUSTOM) is None:
self.api.show_error(
self.name,
'Symbols for CUSTOM variant need to be loaded for mGBA bridge to work.'
)
return
self.dock = BridgeDock(self.api.main_window, self.api)
self.api.main_window.addDockWidget(Qt.LeftDockWidgetArea, self.dock)
def slot_extract_data(self, text: str) -> None:
if self.symbols is None:
# First need to load symbols
self.symbols = get_symbol_database().get_symbols(RomVariant.CUSTOM)
if self.symbols is None:
self.server_worker.slot_extracted_data({
'status':
'error',
'text':
'No symbols for rom CUSTOM loaded'
})
return
if self.data_extractor_plugin is None:
self.data_extractor_plugin = get_plugin('data_extractor',
'DataExtractorPlugin')
if self.data_extractor_plugin is None:
self.server_worker.slot_extracted_data({
'status':
'error',
'text':
'Data Extractor plugin not loaded'
})
return
if self.rom is None:
self.rom = get_rom(RomVariant.CUSTOM)
if self.rom is None:
self.server_worker.slot_extracted_data({
'status':
'error',
'text':
'CUSTOM rom could not be loaded'
})
return
try:
result = self.data_extractor_plugin.instance.extract_data(
text, self.symbols, self.rom)
if result is not None:
self.server_worker.slot_extracted_data({
'status': 'ok',
'text': result
})
except Exception as e:
traceback.print_exc()
self.server_worker.slot_extracted_data({
'status': 'error',
'text': str(e)
})
def slot_call_graph(self) -> None:
symbol_database = get_symbol_database()
pointers = get_pointer_database().get_pointers(rom_variant)
if not symbol_database.are_symbols_loaded(rom_variant):
self.api.show_error(
self.name, f'Symbols for {rom_variant} rom are not loaded')
return
symbols = symbol_database.get_symbols(rom_variant)
nodes = set()
edges: List[Edge] = []
for pointer in pointers.get_sorted_pointers():
symbol_from = symbols.get_symbol_at(pointer.address)
symbol_to = symbols.get_symbol_at(pointer.points_to - ROM_OFFSET)
offset_from = pointer.address - symbol_from.address
offset_to = pointer.points_to - ROM_OFFSET - symbol_to.address
nodes.add(symbol_from.name)
nodes.add(symbol_to.name)
edges.append(
Edge(symbol_from.name, offset_from, symbol_to.name, offset_to))
# Print graph
with open('tmp/call_graph.gml', 'w') as f:
f.write('graph [\n')
f.write('directed 1\n')
nodes_with_order = list(nodes)
for i, node in enumerate(nodes_with_order):
f.write('node [\n')
f.write(f'id {i}\n')
f.write(f'label "{node}"\n')
f.write(']\n')
for edge in edges:
f.write('edge [\n')
f.write(f'source {nodes_with_order.index(edge.from_symbol)}\n')
f.write(f'target {nodes_with_order.index(edge.to_symbol)}\n')
f.write(
f'label "{edge.from_symbol} +{edge.from_offset} -> {edge.to_symbol} +{edge.to_offset}"\n'
)
f.write(']\n')
f.write(']\n')
def slot_find_asmfunc(self) -> None:
symbols = get_symbol_database().get_symbols(
RomVariant.CUSTOM) # Symbols for our custom USA rom
nonmatch = self.collect_asm_funcs()
with open('tmp/asm_funcs.html', 'w') as out:
out.write(
'<link rel="stylesheet" href="theme.css" /><script src="sortable.min.js"></script><table class="sortable-theme-slick" data-sortable><thead><tr><th>File</th><th>Function</th><th>Size</th></thead><tbody>'
)
for (file, func) in nonmatch:
symbol = symbols.find_symbol_by_name(func)
size = 0
if symbol is None:
print(f'No symbol found for {func}, maybe static?')
size = '?'
else:
size = symbol.length
out.write(
f'<tr><td>{file}</td><td>{func}</td><td>{size}</td></tr>')
out.write('</tbody></table>')
self.api.show_message(self.name, 'Wrote to tmp/asm_funcs.html')
def slot_script_addr(self, addr: int) -> None:
print('ADDR: ', addr)
if addr == 0:
self.ui.labelCode.setText('No script executed in current context.')
return
rom = get_rom(RomVariant.CUSTOM)
# receive the current instruction pointer
#instruction_pointer = 0x8009b70
#instruction_pointer = 0x8009d42
instruction_pointer = addr
symbols = get_symbol_database().get_symbols(
RomVariant.CUSTOM) # Symbols for our custom USA rom
symbol = symbols.get_symbol_at(instruction_pointer - ROM_OFFSET)
script_name = symbol.name
script_offset = instruction_pointer - ROM_OFFSET - symbol.address
# Find file containing the script
# TODO or statically find all script files?
script_file = None
for root, dirs, files in os.walk(
os.path.join(settings.get_repo_location(), 'data', 'scripts')):
if script_name + '.inc' in files:
script_file = os.path.join(root, script_name + '.inc')
break
# TODO search the file contents for the script
for file in files:
path = os.path.join(root, file)
with open(path, 'r') as f:
if 'SCRIPT_START ' + script_name in f.read():
script_file = path
if script_file is None:
self.ui.labelCode.setText(
f'ERROR: Count not find script file containing {script_name}')
return
self.ui.labelScriptName.setText(script_file)
script_lines = []
with open(script_file, 'r') as file:
script_lines = file.read().split('\n')
# print(script_lines)
# TODO for testing ifdefs: script_0800B200
# print('test')
# print(symbol)
# print(script_offset)
# TODO only disassemble the number of bytes, the actual instructions are not interesting as they are read from the source file.
(_, instructions) = disassemble_script(
rom.get_bytes(symbol.address, symbol.address + symbol.length),
symbol.address)
output = ''
current_instruction = 0
in_correct_script = False
ifdef_stack = [True]
for line in script_lines:
stripped = line.strip()
if stripped.startswith('SCRIPT_START'):
in_correct_script = stripped == 'SCRIPT_START ' + script_name
output += f'{line}\n'
continue
if not in_correct_script or stripped.startswith(
'@') or stripped.endswith(':'):
output += f'{line}\n'
continue
if '.ifdef' in stripped:
if not ifdef_stack[-1]:
ifdef_stack.append(False)
output += f'{line}\n'
continue
# TODO check variant
is_usa = stripped.split(' ')[1] == 'USA'
ifdef_stack.append(is_usa)
output += f'{line}\n'
continue
if '.ifndef' in stripped:
if not ifdef_stack[-1]:
ifdef_stack.append(False)
output += f'{line}\n'
continue
is_usa = stripped.split(' ')[1] == 'USA'
ifdef_stack.append(not is_usa)
output += f'{line}\n'
continue
if '.else' in stripped:
if ifdef_stack[-2]:
# If the outermost ifdef is not true, this else does not change the validiness of this ifdef
ifdef_stack[-1] = not ifdef_stack[-1]
output += f'{line}\n'
continue
if '.endif' in stripped:
ifdef_stack.pop()
output += f'{line}\n'
continue
if not ifdef_stack[-1]:
# Not defined for this variant
output += f'{line}\n'
continue
if current_instruction >= len(instructions):
# TODO maybe even not print additional lines?
output += f'{line}\n'
continue
addr = instructions[current_instruction].addr
prefix = ''
if addr == script_offset:
prefix = '>'
output += f'{addr:03d}| {prefix}{line}\t\n'
current_instruction += 1
if stripped.startswith('SCRIPT_END'):
break
self.ui.labelCode.setText(output)
def __init__(self, dock: HexViewerDock, rom_variant: RomVariant, rom: Rom) -> None:
super().__init__(parent=dock)
self.dock = dock
self.area = dock.ui.hexArea
self.status_bar = dock.ui.labelStatusBar
self.scroll_bar = dock.ui.scrollBar
self.rom_variant = rom_variant
self.rom = rom
self.address_resolver = TrivialAddressResolver()
self.diff_calculator = NoDiffCalculator()
# State TODO put into different class?
self.is_linked = False
self.start_offset = 0
self.cursor = 0
self.selected_bytes = 1
self.display_byte_cache = {} # TODO invalidate this cache if a constraint is added
# Settings # TODO move elsewhere
self.diff_color = QColor(158, 80, 88) # QColor(244, 108, 117)
self.pointer_color = QColor(68, 69, 34)
self.default_annotation_color = QColor(50, 180, 50)
self.default_selection_size = settings.get_default_selection_size()
self.highlight_8_bytes = settings.is_highlight_8_bytes()
self.contextmenu_handlers = []
self.setup_scroll_bar()
self.scroll_bar.valueChanged.connect(self.slot_scroll_bar_changed)
# Connect to all necessary UI signals
self.dock.ui.pushButtonGoto.clicked.connect(self.slot_show_goto_dialog)
self.dock.ui.pushButtonLink.clicked.connect(self.slot_toggle_linked)
# self.dock.ui.scrollBar.valueChanged.connect(self.on_scroll_bar_changed)
self.area.signal_resized.connect(self.slot_on_resize)
self.area.signal_scroll_wheel_changed.connect(
self.slot_scroll_wheel_changed)
self.area.signal_cursor_changed.connect(
self.slot_update_cursor_from_offset)
self.area.signal_selection_updated.connect(
self.slot_update_selection_from_offset)
self.area.signal_key_cursor_pressed.connect(
self.slot_key_cursor_pressed)
self.area.signal_key_selection_pressed.connect(
self.slot_key_selection_pressed)
self.area.signal_context_menu_shown.connect(
self.slot_shot_context_menu)
self.area.signal_show_tooltip_at_offset.connect(
self.slot_show_tooltip_at_offset)
self.area.signal_go_to_pointer_at_offset.connect(
self.slot_go_to_pointer_at)
# Keyboard shortcuts
QShortcut(QKeySequence(Qt.Key_G), self.dock,
self.slot_show_goto_dialog, context=Qt.WidgetWithChildrenShortcut)
QShortcut(QKeySequence(Qt.CTRL + Qt.Key_C), self.dock,
self.copy_selected_bytes, context=Qt.WidgetWithChildrenShortcut)
QShortcut(QKeySequence(Qt.CTRL + Qt.Key_A), self.dock,
self.mark_as_all_pointer, context=Qt.WidgetWithChildrenShortcut)
QShortcut(QKeySequence(Qt.Key_4), self.dock, lambda:self.update_selected_bytes(4),
context=Qt.WidgetWithChildrenShortcut)
QShortcut(QKeySequence(Qt.Key_8), self.dock, lambda:self.update_selected_bytes(8),
context=Qt.WidgetWithChildrenShortcut)
QShortcut(QKeySequence(Qt.Key_F3), self.dock, self.slot_jump_to_next_diff,
context=Qt.WidgetWithChildrenShortcut)
QShortcut(QKeySequence(Qt.Key_Delete), self.dock, self.slot_delete_current_pointer,
context=Qt.WidgetWithChildrenShortcut)
# TODO tmp
QShortcut(QKeySequence(Qt.Key_Tab), self.dock, lambda:(self.update_cursor(self.cursor+5), self.update_selected_bytes(4)),
context=Qt.WidgetWithChildrenShortcut) # Go to next midi command or whatever
QShortcut(QKeySequence(Qt.Key_W), self.dock, lambda:(self.update_cursor(self.cursor+12), self.update_selected_bytes(4)),
context=Qt.WidgetWithChildrenShortcut)
self.pointers: PointerList = None
self.annotations: AnnotationList = None
self.constraints: ConstraintList = None
self.symbols: SymbolList = None
if settings.is_using_constraints():
self.update_pointers()
get_pointer_database().pointers_changed.connect(self.slot_update_pointers)
self.update_annotations()
get_annotation_database().annotations_changed.connect(self.slot_update_annotations)
if settings.is_using_constraints():
self.update_constraints()
get_constraint_database().constraints_changed.connect(self.slot_update_constraints)
self.update_symbols()
get_symbol_database().symbols_changed.connect(self.slot_update_symbols)
self.update_hex_area()
self.status_bar.setText('loaded')
def update_symbols(self):
symbol_database = get_symbol_database()
if symbol_database.are_symbols_loaded(self.rom_variant):
self.symbols = symbol_database.get_symbols(self.rom_variant)
else:
self.symbols = None
def slot_find_pointers(self) -> None:
if self.incbins is None:
#self.api.show_error('Pointer Extractor', 'Need to parse .incbins first')
#return
self.slot_parse_incbins()
symbol_database = get_symbol_database()
if not symbol_database.are_symbols_loaded(RomVariant.USA):
self.api.show_error('Pointer Extractor',
'Symbols for USA rom need to be loaded first')
return
symbols = symbol_database.get_symbols(RomVariant.USA)
pointers = get_pointer_database().get_pointers(RomVariant.USA)
to_extract: Dict[str, SortedKeyList[Pointer]] = {}
for pointer in pointers:
found = self.incbins.at(pointer.address)
if len(found) == 1:
interval = found.pop()
file = interval.data
if not file in to_extract:
to_extract[file] = SortedKeyList(key=lambda x: x.address)
to_extract[file].add(pointer)
# print(hex(pointer.address))
#print(found.pop())
elif len(found) > 1:
print(
f'Found {len(found)} incbins for address {pointer.address}'
)
# Count unextracted pointers
count = 0
for file in to_extract:
print(f'{file}: {len(to_extract[file])}')
count += len(to_extract[file])
self.api.show_message('Pointer Extractor',
f'{count} unextracted pointers found')
print(count)
# Find symbols that unextracted pointers point to
missing_labels = {}
count = 0
for file in to_extract:
for pointer in to_extract[file]:
symbol = symbols.get_symbol_at(pointer.points_to - ROM_OFFSET)
offset = pointer.points_to - ROM_OFFSET - symbol.address
if offset > 1: # Offset 1 is ok for function pointers
print(pointer)
if symbol.file not in missing_labels:
missing_labels[symbol.file] = SortedKeyList(
key=lambda x: x.address)
# Insert Missing label if there is not already one
label = MissingLabel(pointer.points_to - ROM_OFFSET,
symbol.name, offset, symbol.file)
if label not in missing_labels[symbol.file]:
missing_labels[symbol.file].add(label)
count += 1
continue
print(f'{count} missing labels')
for file in missing_labels:
print(f'{file}: {len(missing_labels[file])}')
# Insert labels for incbins
for path in missing_labels:
output_lines = []
labels = missing_labels[path]
next_label = labels.pop(0)
# Try to find source assembly file
asm_path = os.path.join(settings.get_repo_location(),
path.replace('.o', '.s'))
if not os.path.isfile(asm_path):
print(f'Cannot insert labels in {path}')
print(missing_labels[path])
continue
with open(asm_path, 'r') as file:
for line in file:
if next_label is not None and line.strip().startswith(
'.incbin "baserom.gba"'):
arr = line.split(',')
if len(arr) == 3:
addr = int(arr[1], 16)
length = int(arr[2], 16)
while next_label is not None and next_label.address < addr:
print(f'Cannot insert {next_label}')
if len(labels) == 0: # Extracted all labels
next_label = None
break
next_label = labels.pop(0)
continue
while next_label is not None and next_label.address >= addr and next_label.address < addr + length:
# Calculate new incbins
prev_addr = addr
prev_length = next_label.address - addr
after_addr = next_label.address
after_length = addr + length - after_addr
if prev_length > 0:
# print the incbin
output_lines.append(
incbin_line(prev_addr, prev_length))
# Print the label
label_addr = '{0:#010x}'.format(
next_label.address +
ROM_OFFSET).upper().replace('0X', '')
output_lines.append(
f'gUnk_{label_addr}:: @ {label_addr}\n')
addr = after_addr
length = after_length
if len(labels) == 0: # Extracted all labels
next_label = None
break
next_label = labels.pop(0)
continue
if length > 0:
output_lines.append(incbin_line(addr, length))
continue
output_lines.append(line)
while next_label is not None:
# tmp: print label for script
label_addr = '{0:#010x}'.format(next_label.address +
ROM_OFFSET).upper().replace(
'0X', '')
print(f'SCRIPT_START script_{label_addr}')
print(f'at {next_label.symbol}')
#print(f'Could not insert {next_label}')
if len(labels) == 0: # Extracted all labels
next_label = None
break
next_label = labels.pop(0)
print(f'Write {asm_path}')
print(next_label)
with open(asm_path, 'w') as file:
file.writelines(output_lines)
print('Extracting pointers')
# Extract pointers
for path in to_extract:
output_lines = []
pointers = to_extract[path]
next_pointer = pointers.pop(0)
with open(path, 'r') as file:
for line in file:
if next_pointer is not None and line.strip().startswith(
'.incbin "baserom.gba"'):
arr = line.split(',')
if len(arr) == 3:
addr = int(arr[1], 16)
length = int(arr[2], 16)
while next_pointer.address >= addr and next_pointer.address < addr + length:
# Pointer is in this incbin
symbol = symbols.get_symbol_at(
next_pointer.points_to - ROM_OFFSET)
offset = next_pointer.points_to - ROM_OFFSET - symbol.address
if offset > 1:
# Missing label
if len(pointers
) == 0: # Extracted all pointers
next_pointer = None
break
next_pointer = pointers.pop(0)
continue
# Calculate new incbins
prev_addr = addr
prev_length = next_pointer.address - addr
after_addr = next_pointer.address + 4
after_length = addr + length - after_addr
if after_length < 0:
message = f'Pointer at {hex(next_pointer.address)} crosses over from incbin at {hex(addr)}'
print(path)
print(after_length)
print(message)
self.api.show_error(
'Pointer Extractor', message)
return
if prev_length > 0:
# print the incbin
output_lines.append(
incbin_line(prev_addr, prev_length))
# Print the pointer
output_lines.append(
f'\t.4byte {symbol.name}\n')
addr = after_addr
length = after_length
if len(pointers
) == 0: # Extracted all pointers
next_pointer = None
break
next_pointer = pointers.pop(0)
if length > 0:
output_lines.append(incbin_line(addr, length))
continue
output_lines.append(line)
with open(path, 'w') as file:
file.writelines(output_lines)
#print(''.join(output_lines))
self.api.show_message('Pointer Extractor', f'Done extracting pointers')