def slot_remove_pointer(self, pointer: Pointer) -> None:
pointer_addresses = {
pointer.rom_variant: pointer.points_to - ROM_OFFSET
}
virtual_address = self.constraint_manager.to_virtual(pointer.rom_variant, pointer.address)
print(f'remove {pointer}')
remove_pointers = [pointer]
for rom_variant in self.linked_variants:
if rom_variant == pointer.rom_variant:
continue
local_address = self.constraint_manager.to_local(rom_variant, virtual_address)
pointers = get_pointer_database().get_pointers(rom_variant).get_pointers_at(local_address)
if len(pointers) != 1:
continue
pointer_addresses[rom_variant] = pointers[0].points_to - ROM_OFFSET
print(f'remove {pointers[0]}')
remove_pointers.append(pointers[0])
# Find the corresponding constraints to delete
remove_constraints = []
constraints = get_constraint_database().get_constraints()
for constraint in constraints:
if constraint.romA in pointer_addresses and constraint.addressA == pointer_addresses[constraint.romA]:
if constraint.romB in pointer_addresses and constraint.addressB == pointer_addresses[constraint.romB]:
remove_constraints.append(constraint)
print(f'Remove {constraint}')
if QMessageBox.question(self.parent(), 'Remove Pointer', f'Remove {len(remove_pointers)} pointers and {len(remove_constraints)} constraints?') == QMessageBox.Yes:
get_pointer_database().remove_pointers(remove_pointers)
get_constraint_database().remove_constraints(remove_constraints)
def slot_call_graph(self) -> None:
symbol_database = get_symbol_database()
pointers = get_pointer_database().get_pointers(rom_variant)
if not symbol_database.are_symbols_loaded(rom_variant):
self.api.show_error(
self.name, f'Symbols for {rom_variant} rom are not loaded')
return
symbols = symbol_database.get_symbols(rom_variant)
nodes = set()
edges: List[Edge] = []
for pointer in pointers.get_sorted_pointers():
symbol_from = symbols.get_symbol_at(pointer.address)
symbol_to = symbols.get_symbol_at(pointer.points_to - ROM_OFFSET)
offset_from = pointer.address - symbol_from.address
offset_to = pointer.points_to - ROM_OFFSET - symbol_to.address
nodes.add(symbol_from.name)
nodes.add(symbol_to.name)
edges.append(
Edge(symbol_from.name, offset_from, symbol_to.name, offset_to))
# Print graph
with open('tmp/call_graph.gml', 'w') as f:
f.write('graph [\n')
f.write('directed 1\n')
nodes_with_order = list(nodes)
for i, node in enumerate(nodes_with_order):
f.write('node [\n')
f.write(f'id {i}\n')
f.write(f'label "{node}"\n')
f.write(']\n')
for edge in edges:
f.write('edge [\n')
f.write(f'source {nodes_with_order.index(edge.from_symbol)}\n')
f.write(f'target {nodes_with_order.index(edge.to_symbol)}\n')
f.write(
f'label "{edge.from_symbol} +{edge.from_offset} -> {edge.to_symbol} +{edge.to_offset}"\n'
)
f.write(']\n')
f.write(']\n')
def add_new_pointer(self, pointer: Pointer) -> None:
if pointer.points_to < ROM_OFFSET or pointer.points_to > ROM_OFFSET + ROM_SIZE:
QMessageBox.critical(self.dock, 'Add pointer and constraints', f'Address {hex(pointer.points_to)} is not inside the rom.')
return
get_pointer_database().add_pointer(pointer)
def __init__(self, dock: HexViewerDock, rom_variant: RomVariant, rom: Rom) -> None:
super().__init__(parent=dock)
self.dock = dock
self.area = dock.ui.hexArea
self.status_bar = dock.ui.labelStatusBar
self.scroll_bar = dock.ui.scrollBar
self.rom_variant = rom_variant
self.rom = rom
self.address_resolver = TrivialAddressResolver()
self.diff_calculator = NoDiffCalculator()
# State TODO put into different class?
self.is_linked = False
self.start_offset = 0
self.cursor = 0
self.selected_bytes = 1
self.display_byte_cache = {} # TODO invalidate this cache if a constraint is added
# Settings # TODO move elsewhere
self.diff_color = QColor(158, 80, 88) # QColor(244, 108, 117)
self.pointer_color = QColor(68, 69, 34)
self.default_annotation_color = QColor(50, 180, 50)
self.default_selection_size = settings.get_default_selection_size()
self.highlight_8_bytes = settings.is_highlight_8_bytes()
self.contextmenu_handlers = []
self.setup_scroll_bar()
self.scroll_bar.valueChanged.connect(self.slot_scroll_bar_changed)
# Connect to all necessary UI signals
self.dock.ui.pushButtonGoto.clicked.connect(self.slot_show_goto_dialog)
self.dock.ui.pushButtonLink.clicked.connect(self.slot_toggle_linked)
# self.dock.ui.scrollBar.valueChanged.connect(self.on_scroll_bar_changed)
self.area.signal_resized.connect(self.slot_on_resize)
self.area.signal_scroll_wheel_changed.connect(
self.slot_scroll_wheel_changed)
self.area.signal_cursor_changed.connect(
self.slot_update_cursor_from_offset)
self.area.signal_selection_updated.connect(
self.slot_update_selection_from_offset)
self.area.signal_key_cursor_pressed.connect(
self.slot_key_cursor_pressed)
self.area.signal_key_selection_pressed.connect(
self.slot_key_selection_pressed)
self.area.signal_context_menu_shown.connect(
self.slot_shot_context_menu)
self.area.signal_show_tooltip_at_offset.connect(
self.slot_show_tooltip_at_offset)
self.area.signal_go_to_pointer_at_offset.connect(
self.slot_go_to_pointer_at)
# Keyboard shortcuts
QShortcut(QKeySequence(Qt.Key_G), self.dock,
self.slot_show_goto_dialog, context=Qt.WidgetWithChildrenShortcut)
QShortcut(QKeySequence(Qt.CTRL + Qt.Key_C), self.dock,
self.copy_selected_bytes, context=Qt.WidgetWithChildrenShortcut)
QShortcut(QKeySequence(Qt.CTRL + Qt.Key_A), self.dock,
self.mark_as_all_pointer, context=Qt.WidgetWithChildrenShortcut)
QShortcut(QKeySequence(Qt.Key_4), self.dock, lambda:self.update_selected_bytes(4),
context=Qt.WidgetWithChildrenShortcut)
QShortcut(QKeySequence(Qt.Key_8), self.dock, lambda:self.update_selected_bytes(8),
context=Qt.WidgetWithChildrenShortcut)
QShortcut(QKeySequence(Qt.Key_F3), self.dock, self.slot_jump_to_next_diff,
context=Qt.WidgetWithChildrenShortcut)
QShortcut(QKeySequence(Qt.Key_Delete), self.dock, self.slot_delete_current_pointer,
context=Qt.WidgetWithChildrenShortcut)
# TODO tmp
QShortcut(QKeySequence(Qt.Key_Tab), self.dock, lambda:(self.update_cursor(self.cursor+5), self.update_selected_bytes(4)),
context=Qt.WidgetWithChildrenShortcut) # Go to next midi command or whatever
QShortcut(QKeySequence(Qt.Key_W), self.dock, lambda:(self.update_cursor(self.cursor+12), self.update_selected_bytes(4)),
context=Qt.WidgetWithChildrenShortcut)
self.pointers: PointerList = None
self.annotations: AnnotationList = None
self.constraints: ConstraintList = None
self.symbols: SymbolList = None
if settings.is_using_constraints():
self.update_pointers()
get_pointer_database().pointers_changed.connect(self.slot_update_pointers)
self.update_annotations()
get_annotation_database().annotations_changed.connect(self.slot_update_annotations)
if settings.is_using_constraints():
self.update_constraints()
get_constraint_database().constraints_changed.connect(self.slot_update_constraints)
self.update_symbols()
get_symbol_database().symbols_changed.connect(self.slot_update_symbols)
self.update_hex_area()
self.status_bar.setText('loaded')
def update_pointers(self):
if settings.is_using_constraints():
pointer_database = get_pointer_database()
self.pointers = pointer_database.get_pointers(self.rom_variant)
def add_pointers_and_constraints(self, pointer: Pointer) -> bool:
"""
Add a pointer that is the same for all variants and the resulting constraints.
Returns true if the constraint changes the relations between the files.
"""
# Found
new_pointers = [pointer]
new_constraints = []
virtual_address = self.constraint_manager.to_virtual(
pointer.rom_variant, pointer.address)
for variant in self.linked_variants:
if variant != pointer.rom_variant:
address = self.constraint_manager.to_local(
variant, virtual_address)
points_to = get_rom(variant).get_pointer(address)
# Add a corresponding pointer for this variant
new_pointers.append(Pointer(
variant, address, points_to, pointer.certainty, pointer.author, pointer.note))
# Add a constraint for the places that these two pointers are pointing to, as the pointers should be the same
# TODO check that it's actually a pointer into rom
note = f'Pointer at {pointer.rom_variant} {hex(pointer.address)}'
if pointer.note.strip() != '':
note += '\n' + pointer.note
# TODO test that adding the added constraints are not invalid
# TODO It might be that a constraint that is added with the new_constraints invalidates some other newly added
# constraint which then would need to be enabled. Need to test for all new_constraints whether they are actually still valid after adding them to the constraint manager?
enabled = self.constraint_manager.to_virtual(
pointer.rom_variant, pointer.points_to-ROM_OFFSET) != self.constraint_manager.to_virtual(variant, points_to-ROM_OFFSET)
print(f'Add constraint {enabled}')
new_constraints.append(Constraint(pointer.rom_variant, pointer.points_to-ROM_OFFSET,
variant, points_to-ROM_OFFSET, pointer.certainty, pointer.author, note, enabled))
# Show dialog if one constraint was new
one_enabled = False
for constraint in new_constraints:
if constraint.enabled:
one_enabled = True
break
if one_enabled:
# TODO we cannot be sure yet that the one enabled constraint does not interfere with the disabled constraint,
# so just enable all constraints again (and disable them later via the constraint cleaner plugin)
for constraint in new_constraints:
constraint.enabled = True
# Check whether the new constraint is invalid
constraint_manager = ConstraintManager({RomVariant.USA, RomVariant.DEMO, RomVariant.EU, RomVariant.JP, RomVariant.DEMO_JP, RomVariant.CUSTOM, RomVariant.CUSTOM_EU, RomVariant.CUSTOM_JP, RomVariant.CUSTOM_DEMO_USA, RomVariant.CUSTOM_DEMO_JP})
constraint_manager.add_all_constraints(
get_constraint_database().get_constraints())
try:
constraint_manager.add_all_constraints(new_constraints)
except InvalidConstraintError as e:
raise e
print('Adding to database')
pointer_database = get_pointer_database()
pointer_database.add_pointers(new_pointers)
constraint_database = get_constraint_database()
constraint_database.add_constraints(new_constraints)
return one_enabled
def process(self) -> None:
try:
print('start')
# Load shifted rom
rom_original = get_rom(RomVariant.USA)
print('Load shifted')
rom_path = path.join(settings.get_repo_location(), 'tmc.gba')
print(rom_path)
if not path.isfile(rom_path):
self.signal_fail.emit(f'Shifted rom expected at {rom_path}')
return
rom_shifted = Rom(rom_path)
print('Shifted rom loaded')
pointerlist = get_pointer_database().get_pointers(RomVariant.USA)
END_OF_USED_DATA = 0xde7da4
errors = []
locations = []
shift_location = 0x108
shift_length = 0x10000
take_long_time = True
progress = 0
for i in range(END_OF_USED_DATA):
orig = rom_original.get_byte(i)
shifted_i = i
if i >= shift_location:
#print('SHIFT')
shifted_i += shift_length
#print(i, shifted_i)
shifted = rom_shifted.get_byte(shifted_i)
if orig != shifted:
pointers = pointerlist.get_pointers_at(i)
# Test if pointer
if len(pointers) > 0:
assert shifted == orig + 1
# TODO parse the full pointer
continue
print(f'{hex(i-2)}\t{orig}\t{shifted}')
errors.append((i, orig, shifted))
locations.append(i)
#self.signal_fail.emit(f'Failed at {hex(i)}: {orig} {shifted}')
#break
else:
if take_long_time:
if rom_original.get_byte(i + 1) != 0x8:
# Certainly not a pointer here
continue
pointers = pointerlist.get_pointers_at(i)
if len(pointers) > 0:
if pointers[0].address == i - 2:
errors.append((i, orig, shifted))
locations.append(i)
print(f'missing shift at {hex(i-2)}')
#if len(pointers) > 0:
# TODO test that pointer was shifted
#pass
new_progress = i * 100 // END_OF_USED_DATA
if new_progress != progress:
progress = new_progress
self.signal_progress.emit(new_progress)
if len(errors) == 0:
self.signal_done.emit()
else:
self.signal_locations.emit(locations)
self.signal_fail.emit(f'{len(errors)} errors found.')
except Exception as e:
print(e)
self.signal_fail.emit('Caught exception')
def slot_find_pointers(self) -> None:
if self.incbins is None:
#self.api.show_error('Pointer Extractor', 'Need to parse .incbins first')
#return
self.slot_parse_incbins()
symbol_database = get_symbol_database()
if not symbol_database.are_symbols_loaded(RomVariant.USA):
self.api.show_error('Pointer Extractor',
'Symbols for USA rom need to be loaded first')
return
symbols = symbol_database.get_symbols(RomVariant.USA)
pointers = get_pointer_database().get_pointers(RomVariant.USA)
to_extract: Dict[str, SortedKeyList[Pointer]] = {}
for pointer in pointers:
found = self.incbins.at(pointer.address)
if len(found) == 1:
interval = found.pop()
file = interval.data
if not file in to_extract:
to_extract[file] = SortedKeyList(key=lambda x: x.address)
to_extract[file].add(pointer)
# print(hex(pointer.address))
#print(found.pop())
elif len(found) > 1:
print(
f'Found {len(found)} incbins for address {pointer.address}'
)
# Count unextracted pointers
count = 0
for file in to_extract:
print(f'{file}: {len(to_extract[file])}')
count += len(to_extract[file])
self.api.show_message('Pointer Extractor',
f'{count} unextracted pointers found')
print(count)
# Find symbols that unextracted pointers point to
missing_labels = {}
count = 0
for file in to_extract:
for pointer in to_extract[file]:
symbol = symbols.get_symbol_at(pointer.points_to - ROM_OFFSET)
offset = pointer.points_to - ROM_OFFSET - symbol.address
if offset > 1: # Offset 1 is ok for function pointers
print(pointer)
if symbol.file not in missing_labels:
missing_labels[symbol.file] = SortedKeyList(
key=lambda x: x.address)
# Insert Missing label if there is not already one
label = MissingLabel(pointer.points_to - ROM_OFFSET,
symbol.name, offset, symbol.file)
if label not in missing_labels[symbol.file]:
missing_labels[symbol.file].add(label)
count += 1
continue
print(f'{count} missing labels')
for file in missing_labels:
print(f'{file}: {len(missing_labels[file])}')
# Insert labels for incbins
for path in missing_labels:
output_lines = []
labels = missing_labels[path]
next_label = labels.pop(0)
# Try to find source assembly file
asm_path = os.path.join(settings.get_repo_location(),
path.replace('.o', '.s'))
if not os.path.isfile(asm_path):
print(f'Cannot insert labels in {path}')
print(missing_labels[path])
continue
with open(asm_path, 'r') as file:
for line in file:
if next_label is not None and line.strip().startswith(
'.incbin "baserom.gba"'):
arr = line.split(',')
if len(arr) == 3:
addr = int(arr[1], 16)
length = int(arr[2], 16)
while next_label is not None and next_label.address < addr:
print(f'Cannot insert {next_label}')
if len(labels) == 0: # Extracted all labels
next_label = None
break
next_label = labels.pop(0)
continue
while next_label is not None and next_label.address >= addr and next_label.address < addr + length:
# Calculate new incbins
prev_addr = addr
prev_length = next_label.address - addr
after_addr = next_label.address
after_length = addr + length - after_addr
if prev_length > 0:
# print the incbin
output_lines.append(
incbin_line(prev_addr, prev_length))
# Print the label
label_addr = '{0:#010x}'.format(
next_label.address +
ROM_OFFSET).upper().replace('0X', '')
output_lines.append(
f'gUnk_{label_addr}:: @ {label_addr}\n')
addr = after_addr
length = after_length
if len(labels) == 0: # Extracted all labels
next_label = None
break
next_label = labels.pop(0)
continue
if length > 0:
output_lines.append(incbin_line(addr, length))
continue
output_lines.append(line)
while next_label is not None:
# tmp: print label for script
label_addr = '{0:#010x}'.format(next_label.address +
ROM_OFFSET).upper().replace(
'0X', '')
print(f'SCRIPT_START script_{label_addr}')
print(f'at {next_label.symbol}')
#print(f'Could not insert {next_label}')
if len(labels) == 0: # Extracted all labels
next_label = None
break
next_label = labels.pop(0)
print(f'Write {asm_path}')
print(next_label)
with open(asm_path, 'w') as file:
file.writelines(output_lines)
print('Extracting pointers')
# Extract pointers
for path in to_extract:
output_lines = []
pointers = to_extract[path]
next_pointer = pointers.pop(0)
with open(path, 'r') as file:
for line in file:
if next_pointer is not None and line.strip().startswith(
'.incbin "baserom.gba"'):
arr = line.split(',')
if len(arr) == 3:
addr = int(arr[1], 16)
length = int(arr[2], 16)
while next_pointer.address >= addr and next_pointer.address < addr + length:
# Pointer is in this incbin
symbol = symbols.get_symbol_at(
next_pointer.points_to - ROM_OFFSET)
offset = next_pointer.points_to - ROM_OFFSET - symbol.address
if offset > 1:
# Missing label
if len(pointers
) == 0: # Extracted all pointers
next_pointer = None
break
next_pointer = pointers.pop(0)
continue
# Calculate new incbins
prev_addr = addr
prev_length = next_pointer.address - addr
after_addr = next_pointer.address + 4
after_length = addr + length - after_addr
if after_length < 0:
message = f'Pointer at {hex(next_pointer.address)} crosses over from incbin at {hex(addr)}'
print(path)
print(after_length)
print(message)
self.api.show_error(
'Pointer Extractor', message)
return
if prev_length > 0:
# print the incbin
output_lines.append(
incbin_line(prev_addr, prev_length))
# Print the pointer
output_lines.append(
f'\t.4byte {symbol.name}\n')
addr = after_addr
length = after_length
if len(pointers
) == 0: # Extracted all pointers
next_pointer = None
break
next_pointer = pointers.pop(0)
if length > 0:
output_lines.append(incbin_line(addr, length))
continue
output_lines.append(line)
with open(path, 'w') as file:
file.writelines(output_lines)
#print(''.join(output_lines))
self.api.show_message('Pointer Extractor', f'Done extracting pointers')