def main(config_path,
out_dir,
target_path,
modes,
verbose,
ignore_cache=False):
# Load config
with open(config_path) as f:
config = yaml.safe_load(f.read())
options = config.get("options")
options["modes"] = modes
options["verbose"] = verbose
if not out_dir:
out_dir = options.get("out_dir", None)
if not out_dir:
print(
"Error: Output dir not specified as a command line arg or via the config yaml (out_dir)"
)
sys.exit(2)
else:
out_dir = os.path.join(Path(config_path).parent, out_dir)
if not target_path:
target_path = options.get("target_path", None)
if not target_path:
print(
"Error: Target binary path not specified as a command line arg or via the config yaml (target_path)"
)
sys.exit(2)
else:
target_path = os.path.join(out_dir, target_path)
with open(target_path, "rb") as f:
rom_bytes = f.read()
# Create main output dir
Path(out_dir).mkdir(parents=True, exist_ok=True)
symbol_addrs_path = get_symbol_addrs_path(out_dir, options)
undefined_syms_path = get_undefined_syms_path(out_dir, options)
all_symbols = gather_symbols(symbol_addrs_path, undefined_syms_path)
symbol_ranges = [s for s in all_symbols if s.size > 4]
platform = get_platform(options)
processed_segments = []
ld_sections = []
seg_sizes = {}
seg_split = {}
seg_cached = {}
# Load cache
cache_path = get_cache_path(out_dir, options)
try:
with open(cache_path, "rb") as f:
cache = pickle.load(f)
except Exception:
cache = {}
# Initialize segments
all_segments = initialize_segments(options, config_path,
config["segments"])
for segment in all_segments:
if platform == "n64" and type(
segment) == N64SegCode: # remove special-case sometime
segment_symbols, other_symbols = get_segment_symbols(
segment, all_symbols, all_segments)
segment.seg_symbols = segment_symbols
segment.ext_symbols = other_symbols
segment.all_symbols = all_symbols
segment.symbol_ranges = symbol_ranges
segment.check()
typ = segment.type
if segment.type == "bin" and segment.is_name_default():
typ = "unk"
if typ not in seg_sizes:
seg_sizes[typ] = 0
seg_split[typ] = 0
seg_cached[typ] = 0
seg_sizes[typ] += segment.size
if len(segment.errors) == 0:
if segment.should_run():
# Check cache
cached = segment.cache()
if not ignore_cache and cached == cache.get(
segment.unique_id()):
# Cache hit
seg_cached[typ] += 1
else:
# Cache miss; split
cache[segment.unique_id()] = cached
segment.did_run = True
segment.split(rom_bytes, out_dir)
if len(segment.errors) == 0:
processed_segments.append(segment)
seg_split[typ] += 1
log.dot(status=segment.status())
ld_sections.append(segment.get_ld_section())
for segment in processed_segments:
segment.postsplit(processed_segments)
log.dot(status=segment.status())
# Write ldscript
if "ld" in options["modes"] or "all" in options["modes"]:
if verbose:
log.write(f"saving {config['basename']}.ld")
write_ldscript(config['basename'], out_dir, ld_sections, options)
undefined_syms_to_write = [
s for s in all_symbols
if s.referenced and not s.defined and not s.type == "func"
]
undefined_funcs_to_write = [
s for s in all_symbols
if s.referenced and not s.defined and s.type == "func"
]
# Write undefined_funcs_auto.txt
undefined_funcs_auto_path = get_undefined_funcs_auto_path(out_dir, options)
to_write = undefined_funcs_to_write
if len(to_write) > 0:
with open(undefined_funcs_auto_path, "w", newline="\n") as f:
for symbol in to_write:
f.write(f"{symbol.name} = 0x{symbol.vram_start:X};\n")
# write undefined_syms_auto.txt
undefined_syms_auto_path = get_undefined_syms_auto_path(out_dir, options)
to_write = undefined_syms_to_write
if len(to_write) > 0:
with open(undefined_syms_auto_path, "w", newline="\n") as f:
for symbol in to_write:
f.write(f"{symbol.name} = 0x{symbol.vram_start:X};\n")
# print warnings during split/postsplit
for segment in all_segments:
if len(segment.warnings) > 0:
log.write(
f"{Style.DIM}0x{segment.rom_start:06X}{Style.RESET_ALL} {segment.type} {Style.BRIGHT}{segment.name}{Style.RESET_ALL}:"
)
for warn in segment.warnings:
log.write("warning: " + warn, status="warn")
log.write("") # empty line
# Statistics
unk_size = seg_sizes.get("unk", 0)
rest_size = 0
total_size = len(rom_bytes)
for typ in seg_sizes:
if typ != "unk":
rest_size += seg_sizes[typ]
assert (unk_size + rest_size == total_size)
known_ratio = rest_size / total_size
unk_ratio = unk_size / total_size
log.write(
f"Split {fmt_size(rest_size)} ({known_ratio:.2%}) in defined segments")
for typ in seg_sizes:
if typ != "unk":
tmp_size = seg_sizes[typ]
tmp_ratio = tmp_size / total_size
log.write(
f"{typ:>20}: {fmt_size(tmp_size):>8} ({tmp_ratio:.2%}) {Fore.GREEN}{seg_split[typ]} split{Style.RESET_ALL}, {Style.DIM}{seg_cached[typ]} cached"
)
log.write(
f"{'unknown':>20}: {fmt_size(unk_size):>8} ({unk_ratio:.2%}) from unknown bin files"
)
# Save cache
if cache != {}:
if verbose:
print("Writing cache")
with open(cache_path, "wb") as f:
pickle.dump(cache, f)
return 0 # no error
def main(config_path, base_dir, target_path, modes, verbose, use_cache=True):
global config
# Load config
with open(config_path) as f:
config = yaml.load(f.read(), Loader=yaml.SafeLoader)
options.initialize(config, config_path, base_dir, target_path)
options.set("modes", modes)
if verbose:
options.set("verbose", True)
with options.get_target_path().open("rb") as f2:
rom_bytes = f2.read()
if "sha1" in config:
sha1 = hashlib.sha1(rom_bytes).hexdigest()
e_sha1 = config["sha1"]
if e_sha1 != sha1:
log.error(f"sha1 mismatch: expected {e_sha1}, was {sha1}")
# Create main output dir
options.get_base_path().mkdir(parents=True, exist_ok=True)
processed_segments: List[Segment] = []
seg_sizes: Dict[str, int] = {}
seg_split: Dict[str, int] = {}
seg_cached: Dict[str, int] = {}
# Load cache
if use_cache:
try:
with options.get_cache_path().open("rb") as f3:
cache = pickle.load(f3)
if verbose:
log.write(f"Loaded cache ({len(cache.keys())} items)")
except Exception:
cache = {}
else:
cache = {}
# invalidate entire cache if options change
if use_cache and cache.get("__options__") != config.get("options"):
if verbose:
log.write("Options changed, invalidating cache")
cache = {
"__options__": config.get("options"),
}
# Initialize segments
all_segments = initialize_segments(config["segments"])
# Load and process symbols
if options.mode_active("code"):
log.write("Loading and processing symbols")
symbols.initialize(all_segments)
# Resolve raster/palette siblings
if options.mode_active("img"):
palettes.initialize(all_segments)
# Scan
log.write("Starting scan")
for segment in all_segments:
typ = segment.type
if segment.type == "bin" and segment.is_name_default():
typ = "unk"
if typ not in seg_sizes:
seg_sizes[typ] = 0
seg_split[typ] = 0
seg_cached[typ] = 0
seg_sizes[typ] += 0 if segment.size is None else segment.size
if segment.should_scan():
# Check cache but don't write anything
if use_cache:
if segment.cache() == cache.get(segment.unique_id()):
continue
if segment.needs_symbols:
segment_symbols, other_symbols = get_segment_symbols(
segment, all_segments)
segment.given_seg_symbols = segment_symbols
segment.given_ext_symbols = other_symbols
segment.did_run = True
segment.scan(rom_bytes)
processed_segments.append(segment)
seg_split[typ] += 1
log.dot(status=segment.status())
# Split
log.write("Starting split")
for segment in all_segments:
if use_cache:
cached = segment.cache()
if cached == cache.get(segment.unique_id()):
# Cache hit
seg_cached[typ] += 1
continue
else:
# Cache miss; split
cache[segment.unique_id()] = cached
if segment.should_split():
segment.split(rom_bytes)
log.dot(status=segment.status())
if options.mode_active("ld"):
global linker_writer
linker_writer = LinkerWriter()
for segment in all_segments:
linker_writer.add(segment)
linker_writer.save_linker_script()
linker_writer.save_symbol_header()
# Write undefined_funcs_auto.txt
to_write = [
s for s in symbols.all_symbols
if s.referenced and not s.defined and not s.dead and s.type == "func"
]
if len(to_write) > 0:
with open(options.get_undefined_funcs_auto_path(), "w",
newline="\n") as f:
for symbol in to_write:
f.write(f"{symbol.name} = 0x{symbol.vram_start:X};\n")
# write undefined_syms_auto.txt
to_write = [
s for s in symbols.all_symbols if s.referenced and not s.defined
and not s.dead and not s.type == "func"
]
if len(to_write) > 0:
with open(options.get_undefined_syms_auto_path(), "w",
newline="\n") as f:
for symbol in to_write:
f.write(f"{symbol.name} = 0x{symbol.vram_start:X};\n")
# print warnings during split
for segment in all_segments:
if len(segment.warnings) > 0:
log.write(
f"{Style.DIM}0x{segment.rom_start:06X}{Style.RESET_ALL} {segment.type} {Style.BRIGHT}{segment.name}{Style.RESET_ALL}:"
)
for warn in segment.warnings:
log.write("warning: " + warn, status="warn")
log.write("") # empty line
# Statistics
do_statistics(seg_sizes, rom_bytes, seg_split, seg_cached)
# Save cache
if cache != {} and use_cache:
if verbose:
log.write("Writing cache")
with open(options.get_cache_path(), "wb") as f4:
pickle.dump(cache, f4)
def main(rom_path, config_path, repo_path, modes, verbose, ignore_cache=False):
with open(rom_path, "rb") as f:
rom_bytes = f.read()
# Create main output dir
Path(repo_path).mkdir(parents=True, exist_ok=True)
# Load config
with open(config_path) as f:
config = yaml.safe_load(f.read())
options = config.get("options")
options["modes"] = modes
options["verbose"] = verbose
symbol_addrs_path = get_symbol_addrs_path(repo_path, options)
undefined_syms_path = get_undefined_syms_path(repo_path, options)
provided_symbols, c_func_labels_to_add, special_labels, ranges = gather_symbols(
symbol_addrs_path, undefined_syms_path)
platform = get_platform(options)
processed_segments = []
ld_sections = []
defined_funcs = {}
undefined_funcs = set()
undefined_syms = set()
seg_sizes = {}
seg_split = {}
seg_cached = {}
# Load cache
cache_path = get_cache_path(repo_path, options)
try:
with open(cache_path, "rb") as f:
cache = pickle.load(f)
except Exception:
cache = {}
# Initialize segments
all_segments = initialize_segments(options, config_path,
config["segments"])
for segment in all_segments:
if platform == "n64" and type(
segment) == N64SegCode: # remove special-case sometime
segment.all_functions = defined_funcs
segment.provided_symbols = provided_symbols
segment.special_labels = special_labels
segment.c_labels_to_add = c_func_labels_to_add
segment.symbol_ranges = ranges
segment.check()
tp = segment.type
if segment.type == "bin" and segment.is_name_default():
tp = "unk"
if tp not in seg_sizes:
seg_sizes[tp] = 0
seg_split[tp] = 0
seg_cached[tp] = 0
seg_sizes[tp] += segment.size
if len(segment.errors) == 0:
if segment.should_run():
# Check cache
cached = segment.cache()
if not ignore_cache and cached == cache.get(
segment.unique_id()):
# Cache hit
seg_cached[tp] += 1
else:
# Cache miss; split
cache[segment.unique_id()] = cached
segment.did_run = True
segment.split(rom_bytes, repo_path)
if len(segment.errors) == 0:
processed_segments.append(segment)
if platform == "n64" and type(
segment) == N64SegCode: # edge case
undefined_funcs |= segment.glabels_to_add
defined_funcs = {
**defined_funcs,
**segment.glabels_added
}
undefined_syms |= segment.undefined_syms_to_add
seg_split[tp] += 1
log.dot(status=segment.status())
ld_sections.append(segment.get_ld_section())
for segment in processed_segments:
segment.postsplit(processed_segments)
log.dot(status=segment.status())
# Write ldscript
if "ld" in options["modes"] or "all" in options["modes"]:
if verbose:
log.write(f"saving {config['basename']}.ld")
write_ldscript(config['basename'], repo_path, ld_sections, options)
# Write undefined_funcs_auto.txt
undefined_funcs_auto_path = get_undefined_funcs_auto_path(
repo_path, options)
if verbose:
log.write(f"saving {undefined_funcs_auto_path}")
c_predefined_funcs = set(provided_symbols.keys())
to_write = sorted(undefined_funcs - set(defined_funcs.values()) -
c_predefined_funcs)
if len(to_write) > 0:
with open(undefined_funcs_auto_path, "w", newline="\n") as f:
for line in to_write:
f.write(line + " = 0x" + line.split("_")[1][:8].upper() +
";\n")
# write undefined_syms_auto.txt
undefined_syms_auto_path = get_undefined_syms_auto_path(repo_path, options)
if verbose:
log.write(f"saving {undefined_syms_auto_path}")
to_write = sorted(undefined_syms, key=lambda x: x[0])
if len(to_write) > 0:
with open(undefined_syms_auto_path, "w", newline="\n") as f:
for sym in to_write:
f.write(f"{sym[0]} = 0x{sym[1]:X};\n")
# print warnings and errors during split/postsplit
had_error = False
for segment in all_segments:
if len(segment.warnings) > 0 or len(segment.errors) > 0:
log.write(
f"{Style.DIM}0x{segment.rom_start:06X}{Style.RESET_ALL} {segment.type} {Style.BRIGHT}{segment.name}{Style.RESET_ALL}:"
)
for warn in segment.warnings:
log.write("warning: " + warn, status="warn")
for error in segment.errors:
log.write("error: " + error, status="error")
had_error = True
log.write("") # empty line
if had_error:
return 1
# Statistics
unk_size = seg_sizes.get("unk", 0)
rest_size = 0
total_size = len(rom_bytes)
for tp in seg_sizes:
if tp != "unk":
rest_size += seg_sizes[tp]
assert (unk_size + rest_size == total_size)
known_ratio = rest_size / total_size
unk_ratio = unk_size / total_size
log.write(
f"Split {fmt_size(rest_size)} ({known_ratio:.2%}) in defined segments")
for tp in seg_sizes:
if tp != "unk":
tmp_size = seg_sizes[tp]
tmp_ratio = tmp_size / total_size
log.write(
f"{tp:>20}: {fmt_size(tmp_size):>8} ({tmp_ratio:.2%}) {Fore.GREEN}{seg_split[tp]} split{Style.RESET_ALL}, {Style.DIM}{seg_cached[tp]} cached"
)
log.write(
f"{'unknown':>20}: {fmt_size(unk_size):>8} ({unk_ratio:.2%}) from unknown bin files"
)
# Save cache
if cache != {}:
if verbose:
print("Writing cache")
with open(cache_path, "wb") as f:
pickle.dump(cache, f)
return 0 # no error