Exemple #1
0
def initialize_segments(config_segments: Union[dict, list]) -> List[Segment]:
    seen_segment_names: Set[str] = set()
    ret = []

    for i, seg_yaml in enumerate(config_segments):
        # rompos marker
        if isinstance(seg_yaml, list) and len(seg_yaml) == 1:
            continue

        seg_type = Segment.parse_segment_type(seg_yaml)

        segment_class = Segment.get_class_for_type(seg_type)

        this_start = Segment.parse_segment_start(seg_yaml)
        next_start = Segment.parse_segment_start(config_segments[i + 1])

        segment: Segment = Segment.from_yaml(segment_class, seg_yaml,
                                             this_start, next_start)

        if segment.require_unique_name:
            if segment.name in seen_segment_names:
                log.error(f"segment name '{segment.name}' is not unique")

            seen_segment_names.add(segment.name)

        ret.append(segment)

    return ret
Exemple #2
0
    def split_inner(self, segment, rom_bytes, base_path, generic_out_path):
        if not self.rom_start == self.rom_end:
            if self.type == "c":
                asm_out_dir = Segment.create_split_dir(base_path, os.path.join("asm", "nonmatchings"))

                for func in self.funcs_text:
                    func_name = segment.get_symbol(func, type="func", local_only=True).name

                    if func_name not in self.defined_funcs:
                        segment.create_c_asm_file(self.funcs_text, func, asm_out_dir, self, func_name)

                if not os.path.exists(generic_out_path) and options.get("create_new_c_files", True):
                    segment.create_c_file(self.funcs_text, self, asm_out_dir, base_path, generic_out_path)
            else:
                asm_out_dir = Segment.create_split_dir(base_path, "asm")
                out_lines = self.get_standalone_asm_header()
                for func in self.funcs_text:
                    out_lines.extend(self.funcs_text[func][0])
                    out_lines.append("")

                outpath = Path(os.path.join(asm_out_dir, self.name + ".s"))
                outpath.parent.mkdir(parents=True, exist_ok=True)

                if self.type == "asm" or not os.path.exists(outpath):
                    with open(outpath, "w", newline="\n") as f:
                        f.write("\n".join(out_lines))
Exemple #3
0
    def parse_subsegments(self, segment_yaml) -> List[Segment]:
        base_segments: Dict[str, Segment] = {}
        ret = []
        prev_start: RomAddr = -1

        if "subsegments" not in segment_yaml:
            print(f"Error: Group segment {self.name} is missing a 'subsegments' field")
            sys.exit(2)

        for i, subsection_yaml in enumerate(segment_yaml["subsegments"]):
            # rompos marker
            if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1:
                continue

            typ = Segment.parse_segment_type(subsection_yaml)

            segment_class = Segment.get_class_for_type(typ)

            start = Segment.parse_segment_start(subsection_yaml)
            end = self.rom_end if i == len(segment_yaml["subsegments"]) - 1 else Segment.parse_segment_start(segment_yaml["subsegments"][i + 1])

            if isinstance(start, int) and isinstance(prev_start, int) and start < prev_start:
                print(f"Error: Code segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})")
                sys.exit(1)

            segment: Segment = segment_class(subsection_yaml, start, end)
            segment.sibling = base_segments.get(segment.name, None)
            segment.parent = self
            
            if segment.rom_start != "auto":
                assert isinstance(segment.rom_start, int)
                segment.vram_start = self.rom_to_ram(segment.rom_start)

            # TODO: assumes section order - generalize and stuff
            if self.data_vram_start == None and "data" in segment.type:
                self.data_vram_start = segment.vram_start
            if self.rodata_vram_start == None and "rodata" in segment.type:
                self.data_vram_end = segment.vram_start
                self.rodata_vram_start = segment.vram_start
            if self.rodata_vram_end == None and "bss" in segment.type:
                self.rodata_vram_end = segment.vram_start
                self.bss_vram_start = segment.vram_start

            ret.append(segment)

            # todo change
            if typ in ["c", "asm", "hasm"]:
                base_segments[segment.name] = segment

            prev_start = start

        if self.rodata_vram_start != None and self.rodata_vram_end == None:
            assert self.vram_end is not None
            self.rodata_vram_end = self.vram_end

        return ret
Exemple #4
0
    def add(self, segment: Segment):
        entries = segment.get_linker_entries()
        self.entries.extend(entries)

        self._begin_segment(segment)

        do_next = False
        for i, entry in enumerate(entries):
            if entry.section == "linker":  # TODO: isinstance is preferable
                self._end_block()
                self._begin_segment(entry.segment)

            start = entry.segment.rom_start
            if isinstance(start, int):
                # Create new sections for non-0x10 alignment (hack)
                if start % 0x10 != 0 and i != 0 or do_next:
                    self._end_block()
                    self._begin_segment(entry.segment)
                    do_next = False

                if start % 0x10 != 0 and i != 0:
                    do_next = True

            path_cname = re.sub(
                r"[^0-9a-zA-Z_]", "_",
                str(entry.segment.dir / entry.segment.name) +
                ".".join(entry.object_path.suffixes[:-1]))
            self._write_symbol(path_cname, ".")

            if entry.section != "linker":
                self._writeln(f"{entry.object_path}({entry.section});")

        self._end_segment(segment)
Exemple #5
0
    def split_inner(self, segment, rom_bytes, base_path, generic_out_path):
        if not self.rom_start == self.rom_end:
            asm_out_dir = Segment.create_split_dir(base_path, "asm")

            rom_addr = self.rom_start

            insns = [insn for insn in CodeSubsegment.md.disasm(rom_bytes[self.rom_start : self.rom_end], self.vram_start)]

            funcs = segment.process_insns(insns, rom_addr)

            # TODO: someday make func a subclass of symbol and store this disasm info there too
            for func in funcs:
                segment.get_symbol(func, type="func", create=True, define=True, local_only=True)

            funcs = segment.determine_symbols(funcs)
            segment.gather_jumptable_labels(rom_bytes)
            funcs_text = segment.add_labels(funcs)

            if self.type == "c":
                defined_funcs = set()

                if segment.options.get("do_c_func_detection", True) and os.path.exists(generic_out_path):
                    defined_funcs = CodeSubsegment.get_funcs_defined_in_c(generic_out_path)
                    segment.mark_c_funcs_as_defined(defined_funcs)

                asm_out_dir = Segment.create_split_dir(base_path, os.path.join("asm", "nonmatchings"))

                for func in funcs_text:
                    func_name = segment.get_symbol(func, type="func", local_only=True).name

                    if func_name not in defined_funcs:
                        segment.create_c_asm_file(funcs_text, func, asm_out_dir, self, func_name)

                if not os.path.exists(generic_out_path) and segment.options.get("create_new_c_files", True):
                    segment.create_c_file(funcs_text, self, asm_out_dir, base_path, generic_out_path)
            else:
                out_lines = self.get_asm_header()
                for func in funcs_text:
                    out_lines.extend(funcs_text[func][0])
                    out_lines.append("")

                outpath = Path(os.path.join(asm_out_dir, self.name + ".s"))
                outpath.parent.mkdir(parents=True, exist_ok=True)

                with open(outpath, "w", newline="\n") as f:
                    f.write("\n".join(out_lines))
Exemple #6
0
    def split(self, rom_bytes, base_path):
        out_dir = Segment.create_split_dir(
            base_path, self.options.get("assets_dir", "bin"))

        bin_path = os.path.join(out_dir, self.name + ".bin")
        Path(bin_path).parent.mkdir(parents=True, exist_ok=True)
        with open(bin_path, "wb") as f:
            f.write(rom_bytes[self.rom_start:self.rom_end])
        self.log(f"Wrote {self.name} to {bin_path}")
Exemple #7
0
    def split_inner(self, segment, rom_bytes, base_path, generic_out_path):
        if not self.type.startswith("."):
            asm_out_dir = Segment.create_split_dir(base_path, os.path.join("asm", "data"))

            outpath = Path(os.path.join(asm_out_dir, self.name + f".{self.type}.s"))
            outpath.parent.mkdir(parents=True, exist_ok=True)

            if self.file_text:
                with open(outpath, "w", newline="\n") as f:
                    f.write(self.file_text)
Exemple #8
0
    def parse_subsegments(self, yaml) -> List[Segment]:
        ret: List[Segment] = []

        if not yaml or "subsegments" not in yaml:
            return ret

        prev_start: RomAddr = -1

        for i, subsection_yaml in enumerate(yaml["subsegments"]):
            # End of previous segment
            if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1:
                continue

            typ = Segment.parse_segment_type(subsection_yaml)
            start = Segment.parse_segment_start(subsection_yaml)

            segment_class = Segment.get_class_for_type(typ)

            end = self.get_next_seg_start(i, yaml["subsegments"])

            if (
                isinstance(start, int)
                and isinstance(prev_start, int)
                and start < prev_start
            ):
                log.error(
                    f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})"
                )

            vram = None
            if start != "auto":
                assert isinstance(start, int)
                vram = self.get_most_parent().rom_to_ram(start)

            segment: Segment = Segment.from_yaml(
                segment_class, subsection_yaml, start, end, vram
            )
            segment.parent = self

            ret.append(segment)
            prev_start = start

        return ret
Exemple #9
0
    def split(self, rom_bytes, base_path):
        out_dir = Segment.create_split_dir(base_path, "asm")

        encoding = self.options.get("header_encoding", "ASCII")

        header_lines = []
        header_lines.append(f".section .{self.name}, \"a\"\n")
        header_lines.append(
            self.get_line("word", rom_bytes[0x00:0x04],
                          "PI BSB Domain 1 register"))
        header_lines.append(
            self.get_line("word", rom_bytes[0x04:0x08], "Clockrate setting"))
        header_lines.append(
            self.get_line("word", rom_bytes[0x08:0x0C], "Entrypoint address"))
        header_lines.append(
            self.get_line("word", rom_bytes[0x0C:0x10], "Revision"))
        header_lines.append(
            self.get_line("word", rom_bytes[0x10:0x14], "Checksum 1"))
        header_lines.append(
            self.get_line("word", rom_bytes[0x14:0x18], "Checksum 2"))
        header_lines.append(
            self.get_line("word", rom_bytes[0x18:0x1C], "Unknown 1"))
        header_lines.append(
            self.get_line("word", rom_bytes[0x1C:0x20], "Unknown 2"))
        header_lines.append(
            ".ascii \"" +
            rom_bytes[0x20:0x34].decode(encoding).strip().ljust(20) +
            "\" /* Internal name */")
        header_lines.append(
            self.get_line("word", rom_bytes[0x34:0x38], "Unknown 3"))
        header_lines.append(
            self.get_line("word", rom_bytes[0x38:0x3C], "Cartridge"))
        header_lines.append(
            self.get_line("ascii", rom_bytes[0x3C:0x3E], "Cartridge ID"))
        header_lines.append(
            self.get_line("ascii", rom_bytes[0x3E:0x3F], "Country code"))
        header_lines.append(
            self.get_line("byte", rom_bytes[0x3F:0x40], "Version"))
        header_lines.append("")

        s_path = os.path.join(out_dir, self.name + ".s")
        Path(s_path).parent.mkdir(parents=True, exist_ok=True)
        with open(s_path, "w", newline="\n") as f:
            f.write("\n".join(header_lines))
        self.log(f"Wrote {self.name} to {s_path}")
Exemple #10
0
    def handle_alls(self, segs, base_segs) -> bool:
        for i, elem in enumerate(segs):
            if elem.type.startswith("all_"):
                alls = []

                rep_type = f".{elem.type[4:]}"
                replace_class = Segment.get_class_for_type(rep_type)

                for base in base_segs.items():
                    rep = replace_class("auto", "auto", rep_type, base[0], "auto", extract=False)
                    rep.sibling = base[1]
                    rep.parent = self
                    alls.append(rep)

                del segs[i]
                segs[i:i] = alls
                return True
        return False
Exemple #11
0
    def handle_alls(self, segs: List[Segment], base_segs) -> bool:
        for i, elem in enumerate(segs):
            if elem.type.startswith("all_"):
                alls = []

                rep_type = f"{elem.type[4:]}"
                replace_class = Segment.get_class_for_type(rep_type)

                for base in base_segs.items():
                    if isinstance(elem.rom_start, int) and isinstance(
                        self.rom_start, int
                    ):
                        # Shoddy rom to ram
                        vram_start = elem.rom_start - self.rom_start + self.vram_start
                    else:
                        vram_start = "auto"
                    rep: Segment = replace_class(
                        rom_start=elem.rom_start,
                        rom_end=elem.rom_end,
                        type=rep_type,
                        name=base[0],
                        vram_start=vram_start,
                        extract=False,
                        given_subalign=self.given_subalign,
                        exclusive_ram_id=self.get_exclusive_ram_id(),
                        given_dir=self.given_dir,
                        symbol_name_format=self.symbol_name_format,
                        symbol_name_format_no_rom=self.symbol_name_format_no_rom,
                        args=[],
                        yaml={},
                    )
                    rep.sibling = base[1]
                    rep.parent = self
                    alls.append(rep)

                # Insert alls into segs at i
                del segs[i]
                segs[i:i] = alls
                return True
        return False
Exemple #12
0
    def handle_alls(self, segs: List[Segment], base_segs) -> bool:
        for i, elem in enumerate(segs):
            if elem.type.startswith("all_"):
                alls = []

                rep_type = f"{elem.type[4:]}"
                replace_class = Segment.get_class_for_type(rep_type)

                for base in base_segs.items():
                    if isinstance(elem.rom_start, int):
                        # Shoddy rom to ram
                        vram_start = elem.rom_start - self.rom_start + self.vram_start
                    else:
                        vram_start = "auto"
                    rep = replace_class(
                        elem.rom_start,
                        elem.rom_end,
                        rep_type,
                        base[0],
                        vram_start,
                        False,
                        self.given_subalign,
                        self.given_is_overlay,
                        self.given_dir,
                        [],
                        {},
                    )
                    rep.sibling = base[1]
                    rep.parent = self
                    alls.append(rep)

                # Insert alls into segs at i
                del segs[i]
                segs[i:i] = alls
                return True
        return False
Exemple #13
0
    def add(self, segment: Segment):
        entries = segment.get_linker_entries()
        self.entries.extend(entries)

        self._begin_segment(segment)

        seg_name = get_segment_cname(segment)

        self._write_symbol(f"{seg_name}_TEXT_START", ".")

        force_new_section = False
        text_ended = False
        data_started = False
        data_ended = False
        bss_started = False
        cur_section = None

        for i, entry in enumerate(entries):
            cur_section = entry.section

            if cur_section == "linker":  # TODO: isinstance is preferable
                self._end_block()
                self._begin_segment(entry.segment)
                continue
            elif cur_section == "linker_offset":
                self._write_symbol(
                    f"{get_segment_cname(entry.segment)}_OFFSET",
                    f". - {get_segment_cname(segment)}_ROM_START")
                continue

            # text/data/bss START/END labels
            if not data_started and ("data" in cur_section
                                     or "rodata" in cur_section):
                if not text_ended:
                    text_ended = True
                    self._write_symbol(f"{seg_name}_TEXT_END", ".")

                data_started = True
                self._write_symbol(f"{seg_name}_DATA_START", ".")
            elif data_started and not data_ended and "data" not in cur_section and "rodata" not in cur_section:
                data_ended = True
                self._write_symbol(f"{seg_name}_DATA_END", ".")

                if not bss_started and i < (
                        len(entries) - 1) and "bss" in entries[i + 1].section:
                    bss_started = True
                    self._write_symbol(f"{seg_name}_BSS_START", ".")
            elif not bss_started and "bss" in cur_section:
                bss_started = True
                self._write_symbol(f"{seg_name}_BSS_START", ".")

            if options.get("enable_ld_alignment_hack", False):
                start = entry.segment.rom_start
                if isinstance(start, int):
                    # Create new sections for non-subalign alignment (hack)
                    if start % 0x10 != 0 and i != 0 or force_new_section:
                        self._end_block()
                        self._begin_segment(entry.segment, mid_segment=True)
                        force_new_section = False

                    if start % 0x10 != 0 and i != 0:
                        force_new_section = True

            if entry.object_path and cur_section == ".data":
                path_cname = re.sub(
                    r"[^0-9a-zA-Z_]", "_",
                    str(entry.segment.dir / entry.segment.name) +
                    ".".join(entry.object_path.suffixes[:-1]))
                self._write_symbol(path_cname, ".")

            self._writeln(f"{entry.object_path}({cur_section});")

        if not text_ended:
            self._write_symbol(f"{seg_name}_TEXT_END", ".")
        if not data_started:
            self._write_symbol(f"{seg_name}_DATA_START", ".")
        if not data_ended:
            self._write_symbol(f"{seg_name}_DATA_END", ".")
        if not bss_started:
            self._write_symbol(f"{seg_name}_BSS_START", ".")
        self._write_symbol(f"{seg_name}_BSS_END", ".")

        self._end_segment(segment)
Exemple #14
0
    def add(self, segment: Segment):
        entries = segment.get_linker_entries()
        self.entries.extend(entries)

        self._begin_segment(segment)

        seg_name = get_segment_cname(segment)

        section_labels = [
            LinkerSection(l) for l in options.ld_section_labels()
            if l in options.get_section_order()
        ]

        force_new_section = False
        cur_section = None

        for i, entry in enumerate(entries):
            cur_section = entry.section

            if cur_section == "linker":  # TODO: isinstance is preferable
                self._end_block()
                self._begin_segment(entry.segment)
                continue
            elif cur_section == "linker_offset":
                self._write_symbol(
                    f"{get_segment_cname(entry.segment)}_OFFSET",
                    f". - {get_segment_cname(segment)}_ROM_START",
                )
                continue

            for i, section in enumerate(section_labels):
                if not section.started and section.name == cur_section:
                    if i > 0:
                        if not section_labels[i - 1].ended:
                            section_labels[i - 1].ended = True
                            self._write_symbol(
                                f"{seg_name}{section_labels[i - 1].name.upper()}_END",
                                ".",
                            )
                    section.started = True
                    self._write_symbol(
                        f"{seg_name}{section.name.upper()}_START", ".")

            if options.enable_ld_alignment_hack():
                start = entry.segment.rom_start
                if isinstance(start, int):
                    # Create new sections for non-subalign alignment (hack)
                    if start % 0x10 != 0 and i != 0 or force_new_section:
                        self._end_block()
                        self._begin_segment(entry.segment, mid_segment=True)
                        force_new_section = False

                    if start % 0x10 != 0 and i != 0:
                        force_new_section = True

            if (entry.object_path and cur_section == ".data"
                    and entry.segment.type != "lib"):
                path_cname = re.sub(
                    r"[^0-9a-zA-Z_]",
                    "_",
                    str(entry.segment.dir / entry.segment.name) +
                    ".".join(entry.object_path.suffixes[:-1]),
                )
                self._write_symbol(path_cname, ".")

            # Write out manual entries for images inside .data segments
            seg = entry.segment
            if isinstance(seg, CommonSegData):
                for subseg in seg.subsegments:
                    if isinstance(subseg, N64SegImg):
                        self._write_symbol(get_segment_cname(subseg),
                                           f"0x{subseg.rom_start:X}")

            self._writeln(f"{entry.object_path}({cur_section});")

        for section in section_labels:
            if section.started and not section.ended:
                self._write_symbol(
                    f"{seg_name}_{dotless_type(section.name).upper()}_END",
                    ".")

        self._end_segment(segment)
Exemple #15
0
    def parse_subsegments(self, segment_yaml) -> List[Segment]:
        base_segments: OrderedDict[str, Segment] = OrderedDict()
        ret = []
        prev_start: RomAddr = -1
        inserts: OrderedDict[str, int] = (
            OrderedDict()
        )  # Used to manually add "all_" types for sections not otherwise defined in the yaml

        found_sections = OrderedDict(
            (s_name, Range()) for s_name in self.section_boundaries
        )  # Stores yaml index where a section was first found

        if "subsegments" not in segment_yaml:
            return []

        # Mark any manually added dot types
        if options.auto_all_sections():
            cur_section = None

            for i, subsection_yaml in enumerate(segment_yaml["subsegments"]):
                # rompos marker
                if isinstance(subsection_yaml,
                              list) and len(subsection_yaml) == 1:
                    if cur_section is not None:
                        # End the current section
                        found_sections[cur_section].end = i
                        cur_section = None
                    continue

                typ = Segment.parse_segment_type(subsection_yaml)
                if typ.startswith("all_"):
                    typ = typ[4:]
                if not typ.startswith("."):
                    typ = f".{typ}"

                if typ in found_sections:
                    if cur_section is None:
                        # Starting point
                        found_sections[typ].start = i
                        cur_section = typ
                    else:
                        if cur_section != typ:
                            # We're changing sections
                            if found_sections[cur_section].has_end():
                                log.error(
                                    f"Section {cur_section} end encountered but was already ended earlier!"
                                )
                            if found_sections[typ].has_start():
                                log.error(
                                    f"Section {typ} start encounted but has already started earlier!"
                                )

                            # End the current section
                            found_sections[cur_section].end = i

                            # Start the next section
                            found_sections[typ].start = i
                            cur_section = typ

            if cur_section is not None:
                found_sections[cur_section].end = len(
                    segment_yaml["subsegments"])

            inserts = self.find_inserts(found_sections)

        for i, subsection_yaml in enumerate(segment_yaml["subsegments"]):
            # rompos marker
            if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1:
                continue

            typ = Segment.parse_segment_type(subsection_yaml)
            start = Segment.parse_segment_start(subsection_yaml)

            # Add dummy segments to be expanded later
            if typ.startswith("all_"):
                ret.append(Segment(start, "auto", typ, "", "auto"))
                continue

            segment_class = Segment.get_class_for_type(typ)

            end = self.get_next_seg_start(i, segment_yaml["subsegments"])

            if (isinstance(start, int) and isinstance(prev_start, int)
                    and start < prev_start):
                log.error(
                    f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})"
                )

            vram = None
            if start != "auto":
                assert isinstance(start, int)
                vram = self.get_most_parent().rom_to_ram(start)

            segment: Segment = Segment.from_yaml(segment_class,
                                                 subsection_yaml, start, end,
                                                 vram)
            segment.sibling = base_segments.get(segment.name, None)
            segment.parent = self

            for i, section in enumerate(self.section_order):
                if not self.section_boundaries[section].has_start(
                ) and dotless_type(section) == dotless_type(segment.type):
                    if i > 0:
                        prev_section = self.section_order[i - 1]
                        self.section_boundaries[
                            prev_section].end = segment.vram_start
                    self.section_boundaries[section].start = segment.vram_start

            ret.append(segment)

            # todo change
            if typ in CODE_TYPES:
                base_segments[segment.name] = segment

            prev_start = start

        # Add the automatic all_ sections
        orig_len = len(ret)
        for section in reversed(inserts):
            idx = inserts[section]

            if idx == -1:
                idx = orig_len

            # bss hack TODO maybe rethink
            if section == "bss" and self.vram_start is not None:
                rom_start = self.rom_end
                vram_start = self.vram_start + self.rom_end - self.rom_start
            else:
                rom_start = "auto"
                vram_start = "auto"

            ret.insert(
                idx,
                (Segment(rom_start, "auto", "all_" + section, "", vram_start)))

        check = True
        while check:
            check = self.handle_alls(ret, base_segments)

        # TODO why is this necessary?
        if (self.section_boundaries[".rodata"].has_start()
                and not self.section_boundaries[".rodata"].has_end()):
            assert self.vram_end is not None
            self.section_boundaries[".rodata"].end = self.vram_end

        return ret
Exemple #16
0
 def get_next_seg_start(self, i, subsegment_yamls):
     return (self.rom_end if i == len(subsegment_yamls) -
             1 else Segment.parse_segment_start(subsegment_yamls[i + 1]))