def initialize_segments(config_segments: Union[dict, list]) -> List[Segment]:
seen_segment_names: Set[str] = set()
ret = []
for i, seg_yaml in enumerate(config_segments):
# rompos marker
if isinstance(seg_yaml, list) and len(seg_yaml) == 1:
continue
seg_type = Segment.parse_segment_type(seg_yaml)
segment_class = Segment.get_class_for_type(seg_type)
this_start = Segment.parse_segment_start(seg_yaml)
next_start = Segment.parse_segment_start(config_segments[i + 1])
segment: Segment = Segment.from_yaml(segment_class, seg_yaml,
this_start, next_start)
if segment.require_unique_name:
if segment.name in seen_segment_names:
log.error(f"segment name '{segment.name}' is not unique")
seen_segment_names.add(segment.name)
ret.append(segment)
return ret
def split_inner(self, segment, rom_bytes, base_path, generic_out_path):
if not self.rom_start == self.rom_end:
if self.type == "c":
asm_out_dir = Segment.create_split_dir(base_path, os.path.join("asm", "nonmatchings"))
for func in self.funcs_text:
func_name = segment.get_symbol(func, type="func", local_only=True).name
if func_name not in self.defined_funcs:
segment.create_c_asm_file(self.funcs_text, func, asm_out_dir, self, func_name)
if not os.path.exists(generic_out_path) and options.get("create_new_c_files", True):
segment.create_c_file(self.funcs_text, self, asm_out_dir, base_path, generic_out_path)
else:
asm_out_dir = Segment.create_split_dir(base_path, "asm")
out_lines = self.get_standalone_asm_header()
for func in self.funcs_text:
out_lines.extend(self.funcs_text[func][0])
out_lines.append("")
outpath = Path(os.path.join(asm_out_dir, self.name + ".s"))
outpath.parent.mkdir(parents=True, exist_ok=True)
if self.type == "asm" or not os.path.exists(outpath):
with open(outpath, "w", newline="\n") as f:
f.write("\n".join(out_lines))
def parse_subsegments(self, segment_yaml) -> List[Segment]:
base_segments: Dict[str, Segment] = {}
ret = []
prev_start: RomAddr = -1
if "subsegments" not in segment_yaml:
print(f"Error: Group segment {self.name} is missing a 'subsegments' field")
sys.exit(2)
for i, subsection_yaml in enumerate(segment_yaml["subsegments"]):
# rompos marker
if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1:
continue
typ = Segment.parse_segment_type(subsection_yaml)
segment_class = Segment.get_class_for_type(typ)
start = Segment.parse_segment_start(subsection_yaml)
end = self.rom_end if i == len(segment_yaml["subsegments"]) - 1 else Segment.parse_segment_start(segment_yaml["subsegments"][i + 1])
if isinstance(start, int) and isinstance(prev_start, int) and start < prev_start:
print(f"Error: Code segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})")
sys.exit(1)
segment: Segment = segment_class(subsection_yaml, start, end)
segment.sibling = base_segments.get(segment.name, None)
segment.parent = self
if segment.rom_start != "auto":
assert isinstance(segment.rom_start, int)
segment.vram_start = self.rom_to_ram(segment.rom_start)
# TODO: assumes section order - generalize and stuff
if self.data_vram_start == None and "data" in segment.type:
self.data_vram_start = segment.vram_start
if self.rodata_vram_start == None and "rodata" in segment.type:
self.data_vram_end = segment.vram_start
self.rodata_vram_start = segment.vram_start
if self.rodata_vram_end == None and "bss" in segment.type:
self.rodata_vram_end = segment.vram_start
self.bss_vram_start = segment.vram_start
ret.append(segment)
# todo change
if typ in ["c", "asm", "hasm"]:
base_segments[segment.name] = segment
prev_start = start
if self.rodata_vram_start != None and self.rodata_vram_end == None:
assert self.vram_end is not None
self.rodata_vram_end = self.vram_end
return ret
def add(self, segment: Segment):
entries = segment.get_linker_entries()
self.entries.extend(entries)
self._begin_segment(segment)
do_next = False
for i, entry in enumerate(entries):
if entry.section == "linker": # TODO: isinstance is preferable
self._end_block()
self._begin_segment(entry.segment)
start = entry.segment.rom_start
if isinstance(start, int):
# Create new sections for non-0x10 alignment (hack)
if start % 0x10 != 0 and i != 0 or do_next:
self._end_block()
self._begin_segment(entry.segment)
do_next = False
if start % 0x10 != 0 and i != 0:
do_next = True
path_cname = re.sub(
r"[^0-9a-zA-Z_]", "_",
str(entry.segment.dir / entry.segment.name) +
".".join(entry.object_path.suffixes[:-1]))
self._write_symbol(path_cname, ".")
if entry.section != "linker":
self._writeln(f"{entry.object_path}({entry.section});")
self._end_segment(segment)
def split_inner(self, segment, rom_bytes, base_path, generic_out_path):
if not self.rom_start == self.rom_end:
asm_out_dir = Segment.create_split_dir(base_path, "asm")
rom_addr = self.rom_start
insns = [insn for insn in CodeSubsegment.md.disasm(rom_bytes[self.rom_start : self.rom_end], self.vram_start)]
funcs = segment.process_insns(insns, rom_addr)
# TODO: someday make func a subclass of symbol and store this disasm info there too
for func in funcs:
segment.get_symbol(func, type="func", create=True, define=True, local_only=True)
funcs = segment.determine_symbols(funcs)
segment.gather_jumptable_labels(rom_bytes)
funcs_text = segment.add_labels(funcs)
if self.type == "c":
defined_funcs = set()
if segment.options.get("do_c_func_detection", True) and os.path.exists(generic_out_path):
defined_funcs = CodeSubsegment.get_funcs_defined_in_c(generic_out_path)
segment.mark_c_funcs_as_defined(defined_funcs)
asm_out_dir = Segment.create_split_dir(base_path, os.path.join("asm", "nonmatchings"))
for func in funcs_text:
func_name = segment.get_symbol(func, type="func", local_only=True).name
if func_name not in defined_funcs:
segment.create_c_asm_file(funcs_text, func, asm_out_dir, self, func_name)
if not os.path.exists(generic_out_path) and segment.options.get("create_new_c_files", True):
segment.create_c_file(funcs_text, self, asm_out_dir, base_path, generic_out_path)
else:
out_lines = self.get_asm_header()
for func in funcs_text:
out_lines.extend(funcs_text[func][0])
out_lines.append("")
outpath = Path(os.path.join(asm_out_dir, self.name + ".s"))
outpath.parent.mkdir(parents=True, exist_ok=True)
with open(outpath, "w", newline="\n") as f:
f.write("\n".join(out_lines))
def split(self, rom_bytes, base_path):
out_dir = Segment.create_split_dir(
base_path, self.options.get("assets_dir", "bin"))
bin_path = os.path.join(out_dir, self.name + ".bin")
Path(bin_path).parent.mkdir(parents=True, exist_ok=True)
with open(bin_path, "wb") as f:
f.write(rom_bytes[self.rom_start:self.rom_end])
self.log(f"Wrote {self.name} to {bin_path}")
def split_inner(self, segment, rom_bytes, base_path, generic_out_path):
if not self.type.startswith("."):
asm_out_dir = Segment.create_split_dir(base_path, os.path.join("asm", "data"))
outpath = Path(os.path.join(asm_out_dir, self.name + f".{self.type}.s"))
outpath.parent.mkdir(parents=True, exist_ok=True)
if self.file_text:
with open(outpath, "w", newline="\n") as f:
f.write(self.file_text)
def parse_subsegments(self, yaml) -> List[Segment]:
ret: List[Segment] = []
if not yaml or "subsegments" not in yaml:
return ret
prev_start: RomAddr = -1
for i, subsection_yaml in enumerate(yaml["subsegments"]):
# End of previous segment
if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1:
continue
typ = Segment.parse_segment_type(subsection_yaml)
start = Segment.parse_segment_start(subsection_yaml)
segment_class = Segment.get_class_for_type(typ)
end = self.get_next_seg_start(i, yaml["subsegments"])
if (
isinstance(start, int)
and isinstance(prev_start, int)
and start < prev_start
):
log.error(
f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})"
)
vram = None
if start != "auto":
assert isinstance(start, int)
vram = self.get_most_parent().rom_to_ram(start)
segment: Segment = Segment.from_yaml(
segment_class, subsection_yaml, start, end, vram
)
segment.parent = self
ret.append(segment)
prev_start = start
return ret
def split(self, rom_bytes, base_path):
out_dir = Segment.create_split_dir(base_path, "asm")
encoding = self.options.get("header_encoding", "ASCII")
header_lines = []
header_lines.append(f".section .{self.name}, \"a\"\n")
header_lines.append(
self.get_line("word", rom_bytes[0x00:0x04],
"PI BSB Domain 1 register"))
header_lines.append(
self.get_line("word", rom_bytes[0x04:0x08], "Clockrate setting"))
header_lines.append(
self.get_line("word", rom_bytes[0x08:0x0C], "Entrypoint address"))
header_lines.append(
self.get_line("word", rom_bytes[0x0C:0x10], "Revision"))
header_lines.append(
self.get_line("word", rom_bytes[0x10:0x14], "Checksum 1"))
header_lines.append(
self.get_line("word", rom_bytes[0x14:0x18], "Checksum 2"))
header_lines.append(
self.get_line("word", rom_bytes[0x18:0x1C], "Unknown 1"))
header_lines.append(
self.get_line("word", rom_bytes[0x1C:0x20], "Unknown 2"))
header_lines.append(
".ascii \"" +
rom_bytes[0x20:0x34].decode(encoding).strip().ljust(20) +
"\" /* Internal name */")
header_lines.append(
self.get_line("word", rom_bytes[0x34:0x38], "Unknown 3"))
header_lines.append(
self.get_line("word", rom_bytes[0x38:0x3C], "Cartridge"))
header_lines.append(
self.get_line("ascii", rom_bytes[0x3C:0x3E], "Cartridge ID"))
header_lines.append(
self.get_line("ascii", rom_bytes[0x3E:0x3F], "Country code"))
header_lines.append(
self.get_line("byte", rom_bytes[0x3F:0x40], "Version"))
header_lines.append("")
s_path = os.path.join(out_dir, self.name + ".s")
Path(s_path).parent.mkdir(parents=True, exist_ok=True)
with open(s_path, "w", newline="\n") as f:
f.write("\n".join(header_lines))
self.log(f"Wrote {self.name} to {s_path}")
def handle_alls(self, segs, base_segs) -> bool:
for i, elem in enumerate(segs):
if elem.type.startswith("all_"):
alls = []
rep_type = f".{elem.type[4:]}"
replace_class = Segment.get_class_for_type(rep_type)
for base in base_segs.items():
rep = replace_class("auto", "auto", rep_type, base[0], "auto", extract=False)
rep.sibling = base[1]
rep.parent = self
alls.append(rep)
del segs[i]
segs[i:i] = alls
return True
return False
def handle_alls(self, segs: List[Segment], base_segs) -> bool:
for i, elem in enumerate(segs):
if elem.type.startswith("all_"):
alls = []
rep_type = f"{elem.type[4:]}"
replace_class = Segment.get_class_for_type(rep_type)
for base in base_segs.items():
if isinstance(elem.rom_start, int) and isinstance(
self.rom_start, int
):
# Shoddy rom to ram
vram_start = elem.rom_start - self.rom_start + self.vram_start
else:
vram_start = "auto"
rep: Segment = replace_class(
rom_start=elem.rom_start,
rom_end=elem.rom_end,
type=rep_type,
name=base[0],
vram_start=vram_start,
extract=False,
given_subalign=self.given_subalign,
exclusive_ram_id=self.get_exclusive_ram_id(),
given_dir=self.given_dir,
symbol_name_format=self.symbol_name_format,
symbol_name_format_no_rom=self.symbol_name_format_no_rom,
args=[],
yaml={},
)
rep.sibling = base[1]
rep.parent = self
alls.append(rep)
# Insert alls into segs at i
del segs[i]
segs[i:i] = alls
return True
return False
def handle_alls(self, segs: List[Segment], base_segs) -> bool:
for i, elem in enumerate(segs):
if elem.type.startswith("all_"):
alls = []
rep_type = f"{elem.type[4:]}"
replace_class = Segment.get_class_for_type(rep_type)
for base in base_segs.items():
if isinstance(elem.rom_start, int):
# Shoddy rom to ram
vram_start = elem.rom_start - self.rom_start + self.vram_start
else:
vram_start = "auto"
rep = replace_class(
elem.rom_start,
elem.rom_end,
rep_type,
base[0],
vram_start,
False,
self.given_subalign,
self.given_is_overlay,
self.given_dir,
[],
{},
)
rep.sibling = base[1]
rep.parent = self
alls.append(rep)
# Insert alls into segs at i
del segs[i]
segs[i:i] = alls
return True
return False
def add(self, segment: Segment):
entries = segment.get_linker_entries()
self.entries.extend(entries)
self._begin_segment(segment)
seg_name = get_segment_cname(segment)
self._write_symbol(f"{seg_name}_TEXT_START", ".")
force_new_section = False
text_ended = False
data_started = False
data_ended = False
bss_started = False
cur_section = None
for i, entry in enumerate(entries):
cur_section = entry.section
if cur_section == "linker": # TODO: isinstance is preferable
self._end_block()
self._begin_segment(entry.segment)
continue
elif cur_section == "linker_offset":
self._write_symbol(
f"{get_segment_cname(entry.segment)}_OFFSET",
f". - {get_segment_cname(segment)}_ROM_START")
continue
# text/data/bss START/END labels
if not data_started and ("data" in cur_section
or "rodata" in cur_section):
if not text_ended:
text_ended = True
self._write_symbol(f"{seg_name}_TEXT_END", ".")
data_started = True
self._write_symbol(f"{seg_name}_DATA_START", ".")
elif data_started and not data_ended and "data" not in cur_section and "rodata" not in cur_section:
data_ended = True
self._write_symbol(f"{seg_name}_DATA_END", ".")
if not bss_started and i < (
len(entries) - 1) and "bss" in entries[i + 1].section:
bss_started = True
self._write_symbol(f"{seg_name}_BSS_START", ".")
elif not bss_started and "bss" in cur_section:
bss_started = True
self._write_symbol(f"{seg_name}_BSS_START", ".")
if options.get("enable_ld_alignment_hack", False):
start = entry.segment.rom_start
if isinstance(start, int):
# Create new sections for non-subalign alignment (hack)
if start % 0x10 != 0 and i != 0 or force_new_section:
self._end_block()
self._begin_segment(entry.segment, mid_segment=True)
force_new_section = False
if start % 0x10 != 0 and i != 0:
force_new_section = True
if entry.object_path and cur_section == ".data":
path_cname = re.sub(
r"[^0-9a-zA-Z_]", "_",
str(entry.segment.dir / entry.segment.name) +
".".join(entry.object_path.suffixes[:-1]))
self._write_symbol(path_cname, ".")
self._writeln(f"{entry.object_path}({cur_section});")
if not text_ended:
self._write_symbol(f"{seg_name}_TEXT_END", ".")
if not data_started:
self._write_symbol(f"{seg_name}_DATA_START", ".")
if not data_ended:
self._write_symbol(f"{seg_name}_DATA_END", ".")
if not bss_started:
self._write_symbol(f"{seg_name}_BSS_START", ".")
self._write_symbol(f"{seg_name}_BSS_END", ".")
self._end_segment(segment)
def add(self, segment: Segment):
entries = segment.get_linker_entries()
self.entries.extend(entries)
self._begin_segment(segment)
seg_name = get_segment_cname(segment)
section_labels = [
LinkerSection(l) for l in options.ld_section_labels()
if l in options.get_section_order()
]
force_new_section = False
cur_section = None
for i, entry in enumerate(entries):
cur_section = entry.section
if cur_section == "linker": # TODO: isinstance is preferable
self._end_block()
self._begin_segment(entry.segment)
continue
elif cur_section == "linker_offset":
self._write_symbol(
f"{get_segment_cname(entry.segment)}_OFFSET",
f". - {get_segment_cname(segment)}_ROM_START",
)
continue
for i, section in enumerate(section_labels):
if not section.started and section.name == cur_section:
if i > 0:
if not section_labels[i - 1].ended:
section_labels[i - 1].ended = True
self._write_symbol(
f"{seg_name}{section_labels[i - 1].name.upper()}_END",
".",
)
section.started = True
self._write_symbol(
f"{seg_name}{section.name.upper()}_START", ".")
if options.enable_ld_alignment_hack():
start = entry.segment.rom_start
if isinstance(start, int):
# Create new sections for non-subalign alignment (hack)
if start % 0x10 != 0 and i != 0 or force_new_section:
self._end_block()
self._begin_segment(entry.segment, mid_segment=True)
force_new_section = False
if start % 0x10 != 0 and i != 0:
force_new_section = True
if (entry.object_path and cur_section == ".data"
and entry.segment.type != "lib"):
path_cname = re.sub(
r"[^0-9a-zA-Z_]",
"_",
str(entry.segment.dir / entry.segment.name) +
".".join(entry.object_path.suffixes[:-1]),
)
self._write_symbol(path_cname, ".")
# Write out manual entries for images inside .data segments
seg = entry.segment
if isinstance(seg, CommonSegData):
for subseg in seg.subsegments:
if isinstance(subseg, N64SegImg):
self._write_symbol(get_segment_cname(subseg),
f"0x{subseg.rom_start:X}")
self._writeln(f"{entry.object_path}({cur_section});")
for section in section_labels:
if section.started and not section.ended:
self._write_symbol(
f"{seg_name}_{dotless_type(section.name).upper()}_END",
".")
self._end_segment(segment)
def parse_subsegments(self, segment_yaml) -> List[Segment]:
base_segments: OrderedDict[str, Segment] = OrderedDict()
ret = []
prev_start: RomAddr = -1
inserts: OrderedDict[str, int] = (
OrderedDict()
) # Used to manually add "all_" types for sections not otherwise defined in the yaml
found_sections = OrderedDict(
(s_name, Range()) for s_name in self.section_boundaries
) # Stores yaml index where a section was first found
if "subsegments" not in segment_yaml:
return []
# Mark any manually added dot types
if options.auto_all_sections():
cur_section = None
for i, subsection_yaml in enumerate(segment_yaml["subsegments"]):
# rompos marker
if isinstance(subsection_yaml,
list) and len(subsection_yaml) == 1:
if cur_section is not None:
# End the current section
found_sections[cur_section].end = i
cur_section = None
continue
typ = Segment.parse_segment_type(subsection_yaml)
if typ.startswith("all_"):
typ = typ[4:]
if not typ.startswith("."):
typ = f".{typ}"
if typ in found_sections:
if cur_section is None:
# Starting point
found_sections[typ].start = i
cur_section = typ
else:
if cur_section != typ:
# We're changing sections
if found_sections[cur_section].has_end():
log.error(
f"Section {cur_section} end encountered but was already ended earlier!"
)
if found_sections[typ].has_start():
log.error(
f"Section {typ} start encounted but has already started earlier!"
)
# End the current section
found_sections[cur_section].end = i
# Start the next section
found_sections[typ].start = i
cur_section = typ
if cur_section is not None:
found_sections[cur_section].end = len(
segment_yaml["subsegments"])
inserts = self.find_inserts(found_sections)
for i, subsection_yaml in enumerate(segment_yaml["subsegments"]):
# rompos marker
if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1:
continue
typ = Segment.parse_segment_type(subsection_yaml)
start = Segment.parse_segment_start(subsection_yaml)
# Add dummy segments to be expanded later
if typ.startswith("all_"):
ret.append(Segment(start, "auto", typ, "", "auto"))
continue
segment_class = Segment.get_class_for_type(typ)
end = self.get_next_seg_start(i, segment_yaml["subsegments"])
if (isinstance(start, int) and isinstance(prev_start, int)
and start < prev_start):
log.error(
f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})"
)
vram = None
if start != "auto":
assert isinstance(start, int)
vram = self.get_most_parent().rom_to_ram(start)
segment: Segment = Segment.from_yaml(segment_class,
subsection_yaml, start, end,
vram)
segment.sibling = base_segments.get(segment.name, None)
segment.parent = self
for i, section in enumerate(self.section_order):
if not self.section_boundaries[section].has_start(
) and dotless_type(section) == dotless_type(segment.type):
if i > 0:
prev_section = self.section_order[i - 1]
self.section_boundaries[
prev_section].end = segment.vram_start
self.section_boundaries[section].start = segment.vram_start
ret.append(segment)
# todo change
if typ in CODE_TYPES:
base_segments[segment.name] = segment
prev_start = start
# Add the automatic all_ sections
orig_len = len(ret)
for section in reversed(inserts):
idx = inserts[section]
if idx == -1:
idx = orig_len
# bss hack TODO maybe rethink
if section == "bss" and self.vram_start is not None:
rom_start = self.rom_end
vram_start = self.vram_start + self.rom_end - self.rom_start
else:
rom_start = "auto"
vram_start = "auto"
ret.insert(
idx,
(Segment(rom_start, "auto", "all_" + section, "", vram_start)))
check = True
while check:
check = self.handle_alls(ret, base_segments)
# TODO why is this necessary?
if (self.section_boundaries[".rodata"].has_start()
and not self.section_boundaries[".rodata"].has_end()):
assert self.vram_end is not None
self.section_boundaries[".rodata"].end = self.vram_end
return ret
def get_next_seg_start(self, i, subsegment_yamls):
return (self.rom_end if i == len(subsegment_yamls) -
1 else Segment.parse_segment_start(subsegment_yamls[i + 1]))
