def asm(args):
"""Generate the ASM for the metatable and each section.
This operation needs to be performed once, and once again if tables are to
be relocated. To relocate tables, add a third parameter for that table into
the bank names file with it's new flat address, regenerate the metatable
ASM, then reassemble the ROM.
Generated ASM will be printed to console. This portion of the script is
intended to be piped into a file."""
charmap = parse_charmap(args.charmap)
banknames = parse_bank_names(args.banknames)
banknames = extract_metatable_from_rom(args.rom, charmap, banknames, args)
print('SECTION "MainScript Meta Table", ' +
format_sectionaddr_rom(args.metatable_loc))
for bank in banknames:
print("dw " + bank["symbol"])
print("db BANK(" + bank["symbol"] + ')')
print('')
for bank in banknames:
print('SECTION "' + bank["symbol"] + ' Section", ' +
format_sectionaddr_rom(flat(bank["basebank"], bank["baseaddr"])))
print(bank["symbol"] + ':')
print('\tINCBIN "' +
os.path.join(args.output, bank["objname"]).replace("\\", "/") +
'"')
print(bank["symbol"] + '_END')
print('')
def omnibus_bank_split(rowdata, banknames):
"""Given row data from a non-bank-affiliated file, split it into banks.
Returns a dict whose keys are bank IDs and values are that bank's specific
row data."""
out = {}
for bankid, bank in enumerate(banknames):
in_current_bank = False
for row in rowdata:
try:
rowptr = int(row[0][2:], 16)
bankflat = flat(bank["basebank"], bank["baseaddr"])
#This line assumes text blocks do not share an MBC3 bank, which
#is NOT true for the original game. Patched versions do relocate
#the text blocks to occupy one bank each.
in_current_bank = rowptr >= bankflat and rowptr < bankflat + 0x4000
except ValueError:
pass
if in_current_bank:
if bankid not in out.keys():
out[bankid] = []
out[bankid].append(row)
return out
def asm(args):
"""Generate the ASM for string tables.
This operation needs to be performed once, plus
Generated ASM will be printed to console. This portion of the script is
intended to be piped into a file."""
charmap = parse_charmap(args.charmap)
tablenames = parse_tablenames(args.tablenames)
for table in tablenames:
print(
'SECTION "' + table["symbol"] + ' Section", ' +
format_sectionaddr_rom(flat(table["basebank"], table["baseaddr"])))
print(table["symbol"] + '::')
print('\tINCBIN "' +
os.path.join(args.output, table["objname"]).replace("\\", "/") +
'"')
print(table["symbol"] + '_END')
print('')
def rip_msprite_mtable(rom, offset=0x094D, count=9):
"""Rip an entire ROM's metasprite data."""
cloc = rom.tell()
rom.seek(offset)
#The metasprite metatable is oddly organized here.
#Banks in one place, pointers in the other.
banks = []
ptrs = []
for i in range(0, count):
banks.append(CHARA.unpack(rom.read(1))[0])
for i in range(0, count):
ptrs.append(PTR.unpack(rom.read(2))[0])
asmsrc = "SECTION \"MetaSprite metatable\", " + format_sectionaddr_rom(
offset) + "\n"
asmsrc += "MetaspriteBankMetatable::\n"
for bank in banks:
asmsrc += " db BANK(MetaSprite_" + "{0:x}".format(bank) + ")\n"
asmsrc += "MetaspriteAddressMetatable::\n"
for bank, ptr in zip(banks, ptrs):
asmsrc += " dw MetaSprite_" + "{0:x}".format(bank) + "\n"
asmsrc += "\n"
files = {}
for bank, ptr in zip(banks, ptrs):
table_asmsrc, table_files = rip_msprite_table(rom, flat(bank, ptr))
asmsrc += table_asmsrc + "\n"
files.update(table_files)
return (asmsrc, files)
def make_tbl(args):
charmap = parse_charmap(args.charmap)
tablenames = parse_tablenames(args.tablenames)
for table in tablenames:
#Indexes are compiled in a second pass
if table["format"] == "index":
continue
#If filenames are specified, only export banks that are mentioned there
if len(args.filenames) > 0 and table["objname"] not in args.filenames:
continue
print("Compiling " + table["filename"])
#Open and parse the data
with open(os.path.join(args.input, table["filename"]),
"r",
encoding="utf-8") as csvfile:
csvreader = csv.reader(csvfile)
headers = None
rows = []
for row in csvreader:
if headers is None:
headers = row
else:
rows.append(row)
#Determine what column we want
index_col = headers.index("#")
try:
str_col = headers.index(args.language)
except ValueError:
str_col = index_col
#Pack our strings
packed_strings = []
baseaddr = table["baseaddr"]
entries = []
reverse_entries = {}
for i, row in enumerate(rows):
if str_col >= len(row):
print("WARNING: ROW {} IS MISSING IT'S TEXT!!!".format(i))
packed_strings.append(b"")
continue
packed = pack_text(row[str_col],
specials,
charmap[0],
None,
args.window_width,
1,
memory_widths,
wrap=False,
do_not_terminate=True)
if "stride" in table:
if len(packed) > table["stride"]:
print(
"WARNING: Row {} is too long for the current string table stride of {} in table {}."
.format(i, table["stride"], table["filename"]))
packed = packed[0:table["stride"]]
else:
#Pad the string out with E0s.
packed = packed + bytes([0xE0] *
(table["stride"] - len(packed)))
elif b"\xe0" not in packed:
#Any data beyond an E0 is understood to be trash bytes; thus,
#it does not recieve the implicit terminator.
packed = packed + b"\xe0"
packed_strings.append(packed)
reverse_entries[flat(table["basebank"], baseaddr)] = len(entries)
entries.append(flat(table["basebank"], baseaddr))
baseaddr += len(packed)
#Save these for later
table["entries"] = entries
table["reverse_entries"] = reverse_entries
#Write the data out to the object files. We're done here!
if not os.path.exists(
os.path.dirname(os.path.join(args.output, table["objname"]))):
try:
os.makedirs(
os.path.dirname(os.path.join(args.output,
table["objname"])))
except FileExistsError:
pass
with open(os.path.join(args.output, table["objname"]),
"wb") as objfile:
for line in packed_strings:
objfile.write(line)
for table in tablenames:
#Now's the time to compile indexes
if table["format"] != "index":
continue
#If filenames are specified, only export banks that are mentioned there
if len(args.filenames) > 0 and table["objname"] not in args.filenames:
continue
print("Compiling " + table["filename"])
foreign_ptrs = tablenames[table["foreign_id"]]["entries"]
packed_strings = []
entries = []
reverse_entries = {}
#Open and parse the data
with open(os.path.join(args.input, table["filename"]),
"r",
encoding="utf-8") as csvfile:
csvreader = csv.reader(csvfile)
for row in csvreader:
for cell in row:
packed_strings.append(
PTR.pack(foreign_ptrs[int(cell, 10) - 1] % 0x4000 +
0x4000))
#Write the data out to the object files. We're done here!
with open(os.path.join(args.output, table["objname"]),
"wb") as objfile:
for line in packed_strings:
objfile.write(line)
def extract(args):
charmap = parse_charmap(args.charmap)
tablenames = parse_tablenames(args.tablenames)
with open(args.rom, 'rb') as rom:
#Extract a list of pointers each index is expecting
#This is used for trash byte detection later
for table in tablenames:
if table["format"] != "index":
continue
try:
all_ptrs = tablenames[table["foreign_id"]]["expected_ptrs"]
except KeyError:
all_ptrs = []
rom.seek(flat(table["basebank"], table["baseaddr"]))
for i in range(table["count"]):
ptr = PTR.unpack(rom.read(2))[0]
addr = flat(table["basebank"], ptr)
if addr not in all_ptrs:
all_ptrs.append(addr)
all_ptrs.sort()
tablenames[table["foreign_id"]]["expected_ptrs"] = all_ptrs
for table in tablenames:
#Indexes are extracted in a second pass
if table["format"] == "index":
continue
entries = []
reverse_entries = {}
try:
expected_ptrs = table["expected_ptrs"]
except KeyError:
expected_ptrs = None
csvdir = os.path.join(args.input, table["basedir"])
csvpath = os.path.join(args.input, table["filename"])
install_path(csvdir)
with open(csvpath, "w+", encoding="utf-8") as table_csvfile:
csvwriter = csv.writer(table_csvfile)
csvwriter.writerow(["#", args.language])
if table["format"] == "table":
for i in range(table["count"]):
rom.seek(
flat(table["basebank"],
table["baseaddr"] + i * table["stride"]))
reverse_entries[rom.tell()] = len(entries)
entries.append(rom.tell())
data = extract_string(rom, charmap, table["stride"],
expected_ptrs).encode("utf-8")
idx = "{0}".format(i + 1).encode("utf-8")
csvwriter.writerow([idx, data])
elif table["format"] == "block":
rom.seek(flat(table["basebank"], table["baseaddr"]))
for i in range(table["count"]):
reverse_entries[rom.tell()] = len(entries)
entries.append(rom.tell())
data = extract_string(rom, charmap, None,
expected_ptrs).encode("utf-8")
idx = "{0}".format(i + 1).encode("utf-8")
csvwriter.writerow([idx, data])
#Save these for later
table["entries"] = entries
table["reverse_entries"] = reverse_entries
#OK, now we can extract the indexes
for table in tablenames:
if table["format"] != "index":
continue
foreign_ptrs = tablenames[table["foreign_id"]]["reverse_entries"]
rom.seek(flat(table["basebank"], table["baseaddr"]))
csvdir = os.path.join(args.input, table["basedir"])
csvpath = os.path.join(args.input, table["filename"])
install_path(csvdir)
with open(csvpath, "w+", encoding="utf-8") as table_csvfile:
csvwriter = csv.writer(table_csvfile)
pretty_row_length = math.ceil(math.sqrt(table["count"]))
cur_row = []
for i in range(table["count"]):
ptr = PTR.unpack(rom.read(2))[0]
addr = flat(table["basebank"], ptr)
cur_row.append("{0}".format(foreign_ptrs[addr] +
1).encode("utf-8"))
if len(cur_row) >= pretty_row_length:
csvwriter.writerow(cur_row)
cur_row = []
if len(cur_row) > 0:
csvwriter.writerow(cur_row)
def decompress_bank(rom, offset=None):
"""Decompress an entire table of compressed data.
First return value is an array of data, individual data items match the
return value of decompress_tilemap. Data is ordered in the same order that
the ROM orders it. Second return value is the extracted table, represented
as indexes into the data array.
If offset is given, ROM will be read from that position. Your existing ROM
position will be preserved."""
if offset is None:
offset = rom.tell()
last = rom.tell()
rom.seek(offset)
ret_dat = []
order_dat = []
ptr_table = [] #List of PTRs as specified in the table
ptr_data = [] #List of PTRs containing data, some unused
ptr_data_org = {} #Decompressed data
ptr_data_index = {}
#The table consists of multiple pointers followed immediately by compressed
#tilemap data. There is no explicit length, so we discover the length of
#the pointer table by reading and decompressing pointers until we overrun
#already-decompressed data. Oh, and they don't come in a specified order
#either.
first_data_ptr = 0xFFFF
last_data_ptr = 0x0000
last_ptr = None
start_ptr, this_bank = banked(offset)
#Extract the table first.
while banked(rom.tell())[0] < first_data_ptr:
this_ptr = PTR.unpack(rom.read(2))[0]
ptr_table.append(this_ptr)
if this_ptr <= first_data_ptr:
first_data_ptr = this_ptr
if last_data_ptr <= this_ptr:
last_data_ptr = this_ptr
#Extract the data second
this_ptr = first_data_ptr
while this_ptr <= last_data_ptr:
decomp_data, decomp_len = decompress_tilemap(rom,
flat(this_bank, this_ptr))
ptr_data.append(this_ptr)
ptr_data_org[this_ptr] = decomp_data
this_ptr += decomp_len
#Resolve PTRs into numerical indexes and return them.
for data_index, data_ptr in enumerate(ptr_data):
ret_dat.append(ptr_data_org[data_ptr])
ptr_data_index[data_ptr] = data_index
for table_index, table_ptr in enumerate(ptr_table):
order_dat.append(ptr_data_index[table_ptr])
rom.seek(last)
return ret_dat, order_dat
def rip_msprite_table(rom, offset=None):
"""Given a ROM and an offset, rip a metatable of metasprites. Ripped data
will be returned as an array of ASM and a dict listing files to save."""
cloc = rom.tell()
if offset is not None:
rom.seek(offset)
else:
offset = cloc
table_ptr, bank = banked(offset)
ptrs = []
end = 0x7FFF
cur = table_ptr
table_ptr_list = []
ripped_sprites = {}
while cur < end:
offset_ptr = PTR.unpack(rom.read(2))[0]
cur += 2
end = min(offset_ptr, end)
if offset_ptr not in list(ripped_sprites.keys()):
try:
ripped_sprites[offset_ptr] = rip_msprite(
rom, flat(bank, offset_ptr))
except KeyError:
#If a metasprite points at undecipherable data, assume it's a
#dummy pointer and don't generate a metasprite for it, since
#it's clearly run off the end of the table.
ripped_sprites[offset_ptr] = []
table_ptr_list.append(offset_ptr)
#Now that we have the tables and their data, generate the ASM for it...
data_asmsrc = ""
table_asmsrc = ""
files = {}
table_symbol = "MetaSprite_" + "{0:x}".format(bank)
table_asmsrc += "SECTION \"" + table_symbol + "\", ROMX[$" + "{0:x}".format(
table_ptr) + "], BANK[$" + "{0:x}".format(bank) + "]\n"
table_asmsrc += table_symbol + "::\n"
sorted_ptrs = list(ripped_sprites.keys())
sorted_ptrs.sort()
last_ptrkey_end = 0x0000
for ptrkey in sorted_ptrs:
val = ripped_sprites[ptrkey]
symbol = table_symbol + "_" + "{0:x}".format(ptrkey)
file = "gfx/unknown/metasprite_" + "{0:x}".format(
bank) + "/" + "{0:x}".format(ptrkey) + ".sprite.csv"
binfile = "gfx/unknown/metasprite_" + "{0:x}".format(
bank) + "/" + "{0:x}".format(ptrkey) + ".sprite.bin"
if last_ptrkey_end != ptrkey:
data_asmsrc += "SECTION \"" + symbol + "\", ROMX[$" + "{0:x}".format(
ptrkey) + "], BANK[$" + "{0:x}".format(bank) + "]\n"
data_asmsrc += symbol + "::\n"
#ASSUMPTION: All 0 size metasprites are dummy pointers
if len(val) > 0:
data_asmsrc += " INCBIN \"" + binfile + "\"\n"
data_asmsrc += symbol + "_END::\n"
files[file] = "Y,X,Tile Offset,Attribute Mixing,Attributes\n"
for spritecfg in val:
files[file] += "{0:x},{1:x},{2:x},{3},{4:x}\n".format(
spritecfg['y'], spritecfg['x'], spritecfg['tile'],
spritecfg['attrib_mode'], spritecfg['attribs'])
last_ptrkey_end = ptrkey + 1 + len(val) * 5
for ptr in table_ptr_list:
symbol = table_symbol + "_" + "{0:x}".format(ptr)
table_asmsrc += " dw " + symbol + "\n"
asm_src = table_asmsrc + "\n" + data_asmsrc
rom.seek(cloc)
return (asm_src, files)
def extract(args):
charmap = parse_charmap(args.charmap)
banknames = parse_bank_names(args.banknames)
banknames = extract_metatable_from_rom(args.rom, charmap, banknames, args)
with open(args.rom, 'rb') as rom:
for bank in banknames:
wikitext = ["{|", "|-", "!Pointer", "!" + args.language]
csvdata = [["Pointer", args.language]]
rom.seek(flat(bank["basebank"], bank["baseaddr"]))
addr = bank["baseaddr"]
end = 0x8000
#Autodetect the end/length of the table by finding the lowest
#pointer that isn't stored after an existing pointer
while addr < end:
next_ptr = PTR.unpack(rom.read(2))[0]
#Reject obviously invalid pointers
if (next_ptr < addr or next_ptr > 0x7FFF):
break
end = min(end, next_ptr)
addr += 2
tbl_length = (addr - bank["baseaddr"]) // 2
#Actually extract our strings
string = []
#Stores the actual end of the last string, used for alias detection
last_start = 0xFFFF
last_end = 0xFFFF
last_nonaliasing_row = -1
#Also store if a redirected/overflowed row is being extracted
redirected = False
old_loc = None
for i in range(tbl_length):
csvrow = [
"0x{0:x}".format(
flat(bank["basebank"], bank["baseaddr"] + i * 2))
]
wikitext.append("|-")
wikitext.append("|0x{0:x}".format(
flat(bank["basebank"], bank["baseaddr"] + i * 2)))
rom.seek(flat(bank["basebank"], bank["baseaddr"] + i * 2))
read_ptr = PTR.unpack(rom.read(2))[0]
#Attempt to autodetect "holes" in the text data.
next_ptr = PTR.unpack(rom.read(2))[0]
expected_length = next_ptr - read_ptr
if i >= tbl_length - 1:
expected_length = -1 #maximum length by far
#Two different alias detects:
#First, we try to see if this pointer matches another pointer
#in the table.
rom.seek(flat(bank["basebank"], bank["baseaddr"]))
for j in range(i):
if read_ptr == PTR.unpack(rom.read(2))[0]:
#Aliased pointer!
csvrow.append("<ALIAS ROW 0x{0:x}>".format(j))
wikitext.append("|«ALIAS ROW 0x{0:x}»".format(j))
print(
"Pointer at 0x{0:x} fully aliases pointer 0x{1:x}".
format(
flat(bank["basebank"],
bank["baseaddr"] + i * 2),
flat(bank["basebank"],
bank["baseaddr"] + j * 2)))
break
else:
#Second, we try to see if this pointer is in the middle of
#the last string.
#This alias detection breaks when the previous row uses the
#overflow code, so disable it if so.
if i > 0 and read_ptr < last_end - 1 and not redirected:
print(
"Pointer at 0x{0:x} partially aliases previous pointer"
.format(rom.tell() - 2))
csvrow.append(
"<ALIAS ROW 0x{0:x} INTO 0x{1:x}>".format(
last_nonaliasing_row, read_ptr - last_start))
wikitext.append(
"|«ALIAS ROW 0x{0:x} INTO 0x{1:x}»".format(
last_nonaliasing_row, read_ptr - last_start))
continue
read_length = 1
first_read = True
rom.seek(flat(bank["basebank"], read_ptr))
#Now we can initialize these...
redirected = False
old_loc = None
while (rom.tell() % 0x4000 < 0x3FFF or rom.tell() == flat(
bank["basebank"], bank["baseaddr"])):
next_chara = CHARA.unpack(rom.read(1))[0]
while (rom.tell() % 0x4000 < 0x3FFF or rom.tell() ==
flat(bank["basebank"], bank["baseaddr"])) and (
read_length <= expected_length or first_read
or redirected
) and next_chara != 0xE0: #E0 is end-of-string
if next_chara < 0xE0 and next_chara in charmap[
1]: #Control codes are the E0 block
string.append(charmap[1][next_chara])
elif next_chara in reverse_specials and specials[
reverse_specials[next_chara]].redirect:
#Redirecting opcodes are transparently removed from the extracted text.
this_special = specials[
reverse_specials[next_chara]]
if this_special.bts:
read_length += this_special.bts
fmt = "<" + ("", "B",
"H")[this_special.bts]
word = struct.unpack(
fmt, rom.read(this_special.bts))[0]
if word < 0x4000 or word > 0x7FFF:
#Overflowing into RAM is illegal - use the jump opcode.
#Overflowing into ROM0 is technically not illegal, but
#unorthodox enough that we're going to disallow it.
string.append(
format_literal(this_special.byte))
string.append(
format_literal(
word & 0xFF, charmap[1]))
string.append(
format_literal(
word >> 8, charmap[1]))
else:
#We need to do this right now to avoid breaking hole detection
old_loc = rom.tell()
read_length = rom.tell() - flat(
bank["basebank"], read_ptr)
rom.seek(flat(args.overflow_bank,
word))
redirected = True
else:
raise RuntimeError(
"Invalid specials dictionary. Redirecting special character is missing bts."
)
elif next_chara in reverse_specials:
#This must be the work of an 「ＥＮＥＭＹ ＳＴＡＮＤ」
this_special = specials[
reverse_specials[next_chara]]
if this_special.bts:
read_length += this_special.bts
fmt = "<" + ("", "B",
"H")[this_special.bts]
word = struct.unpack(
fmt, rom.read(this_special.bts))[0]
string.append(
format_control_code(
reverse_specials[next_chara],
word))
else:
string.append(
format_control_code(
reverse_specials[next_chara]))
if this_special.end:
first_read = False
break
#elif next_chara == 0xE2:
#Literal newline
# string.append(u"\n")
else:
#Literal specials
string.append(format_literal(next_chara))
next_chara = CHARA.unpack(rom.read(1))[0]
#Explicitly stop updating read_length if the
#overflow opcode is used. Otherwise we'd think we
#read thousands or negative thousands of chars
if not redirected:
read_length = rom.tell() - flat(
bank["basebank"], read_ptr)
#After the main extraction loop
if read_length >= expected_length:
break
else:
#Detect nulls (spaces) after the end of a string
#and append them to avoid creating a new pointer row
loc = rom.tell()
if redirected:
loc = old_loc
while CHARA.unpack(rom.read(1))[0] == charmap[0][
" "] and read_length < expected_length:
string.append(" ")
loc += 1
read_length += 1
rom.seek(loc) #cleanup
if read_length >= expected_length:
break
else:
#There's a hole in the ROM!
#Disassemble the next string.
print("Inaccessible data found at 0x{0:x}".
format(flat(bank["basebank"], read_ptr)))
csvrow.append("".join(string))
wikitext.append("|" + "".join(string))
string = []
csvdata.append(csvrow)
csvrow = ["(No pointer)"]
wikitext.append("|-")
wikitext.append("|(No pointer)")
read_length += 1
csvrow.append("".join(string))
wikitext.append("|" + "".join(string))
string = []
#Store the actual end pointer for later use.
last_start = read_ptr
last_end = read_ptr + read_length
last_nonaliasing_row = i
csvdata.append(csvrow)
wikitext.append("|-")
wikitext.append("|}")
wikitext = "\n".join(wikitext)
wikidir = os.path.join(args.input, bank["basedir"])
wikipath = os.path.join(args.input, bank["legacy_filename"])
csvpath = os.path.join(args.input, bank["filename"])
install_path(wikidir)
#with open(wikipath, "w+", encoding="utf-8") as bank_wikitext:
#bank_wikitext.write(wikitext)
with open(csvpath, "w+", encoding="utf-8") as bank_csvtext:
csvwriter = csv.writer(bank_csvtext)
for csvrow in csvdata:
csvwriter.writerow(csvrow)
def make_tbl(args):
"""Compile the entire script into a single RGBDS file, and then write it to
disk.
`args` is the arguments passed to the program as processed by `argparse`."""
charmap = parse_charmap(args.charmap)
banknames = parse_bank_names(args.banknames)
overflow_bank_ids = []
if args.language == "Japanese":
metrics = None
else:
metrics_file = open(args.metrics, 'r')
metrics = parse_font_metrics(metrics_file)
#Some CSV files in the patch branch are merged together.
#We'll parse these first and add their row data to each individual bank...
for filename in args.filenames:
with open(filename, "r", encoding="utf-8") as csvfile:
rowdata = parse_csv(csvfile, args.language)
split_rowdata = omnibus_bank_split(rowdata, banknames)
for bankid, rowdata in split_rowdata.items():
banknames[bankid]["textdata"] = rowdata
overflow_strings = {}
bank_sources = {}
for h, bank in enumerate(banknames):
if bank["filename"].startswith("script/overflow") or bank[
"filename"].startswith("script\\overflow"):
#Don't attempt to compile the overflow bank. That's a separate pass
overflow_bank_ids.append(h)
continue
#If filenames are specified, don't bother because we can't do on-demand
#compilation anyway and the makefile gets the path wrong
print("Compiling " + bank["filename"])
#Open and parse the data
if "textdata" not in bank.keys():
with open(os.path.join(args.input, bank["filename"]),
"r",
encoding='utf-8') as csvfile:
bank["textdata"] = parse_csv(csvfile, args.language)
bank_window_width = args.window_width
if "window_width" in list(bank.keys()):
bank_window_width = bank["window_width"]
srcdata, overflow = generate_table_section(bank, bank["textdata"],
charmap, metrics,
bank_window_width)
bank_sources[bank["objname"]] = srcdata
overflow_strings.update(overflow)
number_symbols_exported = 0
overflow_sources = {}
for overflow_bank_id in overflow_bank_ids:
overflow_src = format_section("Overflow Bank %d" % overflow_bank_id,
flat(args.overflow_bank, 0x4000))
overflow_offset = 0
current_symbol_id = 0
break_after_adding = False
for symname, packed_str in overflow_strings.items():
if current_symbol_id < number_symbols_exported:
current_symbol_id += 1
continue
if overflow_offset + len(packed_str) > 0x4000:
number_symbols_exported = current_symbol_id
break
current_symbol_id += 1
overflow_src += format_symbol(symname, True)
overflow_src += format_directives(packed_str)
overflow_offset += len(packed_str)
else:
break_after_adding = True
overflow_sources[overflow_bank_id] = overflow_src
if break_after_adding:
break
with open(args.output_filename, "wb") as srcfile:
for bank_id, bank_source in overflow_sources.items():
srcfile.write(bank_source.encode("utf-8"))
for bank_id, bank_source in bank_sources.items():
srcfile.write(bank_source.encode("utf-8"))
def generate_table_section(bank, rows, charmap, metrics, bank_window_width):
"""Given a bank and string data, generate RGBDS source that assembles to it.
`rows` is assumed to be an iterable yielding 2-tuples, where the first item
in each tuple is the pointer index; and the second item is the string to
pack.
`charmap` is a map of string characters to encoded bytes, used to configure
the text encoding for this particular iteration of Telefang.
`metrics` is the character width of each encoded byte.
`bank_window_width` is the pixel width of the window this particular bank
of strings is expected to fit into.
This function returns a tuple with the following information:
- srcdata: RGBDS source code containing all table section data and symbols.
- overflow: A dict of data which overflows this block. Dictionary keys
correspond to the RGBDS symbols which are expected to be defined in the
overflow bank, while the values are the actual string data to put there."""
table = []
packed_strings = [b""] * len(rows)
baseaddr = bank["baseaddr"]
lastbk = None
last_table_index = 0
last_aliased_row = -1
ptr_col = 0
str_col = 1
wrap_col = 2
for i, row in enumerate(rows):
if "#" in row[ptr_col]:
print("Skipping row {} as it is not a message.".format(i))
continue
if row[ptr_col] != "(No pointer)":
try:
table_idx = (int(row[ptr_col][2:], 16) -
bank["baseaddr"]) % 0x4000 // 2
last_table_index = table_idx
except ValueError:
continue #Skip rows that don't have parsable ptr columns
else:
table_idx = last_table_index
if str_col >= len(row):
print("WARNING: ROW {} IS MISSING IT'S TEXT!!!".format(i))
table.append(baseaddr)
packed = pack_text("/0x{0:X}/".format(table_idx),
specials,
charmap[0],
metrics,
bank_window_width,
2,
memory_widths,
wrap=row[wrap_col])
baseaddr += len(packed)
packed_strings[table_idx] = packed
continue
if row[str_col][:11] == "«ALIAS ROW " or row[
str_col][:11] == "<ALIAS ROW ":
#Aliased string!
split_row = row[str_col][11:-1].split(" ")
if len(split_row) > 1 and split_row[1] == "INTO":
#Partial string alias.
table.append(table[int(split_row[0], 16)] +
int(split_row[2], 16))
packed_strings[table_idx] = b""
else:
table.append(table[int(split_row[0], 16)])
packed_strings[table_idx] = b""
#We don't want to have to try to handle both overflow and
#aliasing at the same time, so don't.
last_aliased_row = table_idx
else:
packed = pack_text(row[str_col],
specials,
charmap[0],
metrics,
bank_window_width,
2,
memory_widths,
wrap=row[wrap_col],
do_not_terminate=row[ptr_col] == "(No pointer)")
packed_strings[
table_idx] += packed #We concat here in case of nopointer rows
if row[ptr_col] != "(No pointer)":
#Yes, some text banks have strings not mentioned in the
#table because screw you.
table.append(baseaddr)
#Sanity check: are our pointer numbers increasing?
nextbk = int(rows[i][ptr_col], 16)
if lastbk != None and nextbk != lastbk + 2:
print("Warning: Pointer " + rows[i][ptr_col] +
" is out of order.")
lastbk = nextbk
else:
print("Warning: Row explicitly marked with no pointer")
baseaddr += len(packed)
#Remove empty strings from packed strings list.
while len(packed_strings) > 0 and packed_strings[-1] == b"":
packed_strings = packed_strings[:-1]
#Moveup pointers to account for the table size
for i, value in enumerate(table):
table[i] = value + len(table) * 2
#Overflow detection + handling
bytes_remaining = 0x4000
for intptr in table:
bytes_remaining -= 2
for line in packed_strings:
bytes_remaining -= len(line)
overflow = {}
if bytes_remaining <= 0:
print("Bank " + bank["filename"] +
" exceeds size of MBC bank limit by 0x{0:x} bytes".format(
abs(bytes_remaining)))
#Compiled bank exceeds the amount of space available in the bank.
#Start assigning strings from the last one forward to be spilled
#into the overflow bank. This reduces their size to 4.
strings_to_spill = 0
string_bytes_saved = 0
for table_idx, spill_string in reversed(list(
enumerate(packed_strings))):
if table_idx == last_aliased_row:
#We can't spill aliased rows, nor do we want to attempt to
#support that usecase, since it would be too much work.
#Instead, stop spilling at the end.
print(
"WARNING: Terminating spills at 0x{0:x} due to aliasing.".
format(table_idx))
print("The resulting object file is invalid.")
print("Please consider removing aliasing rows.")
break
strings_to_spill += 1
string_bytes_saved += len(packed_strings[table_idx]) - 4
if string_bytes_saved + bytes_remaining > 0:
#That's enough! Stop counting bytes, we've spilled enough.
break
#Replace each spilled string with the overflow opcode and a fixup into
#the overflow list.
for table_idx in range(
len(packed_strings) - strings_to_spill, len(packed_strings)):
spill_symbol_name = bank["symbol"] + ("_%d_OVERFLOW" % table_idx)
cur_string = packed_strings[table_idx]
packed_strings[table_idx] = (bytes([0xEC]),
"db BANK(%s)" % spill_symbol_name,
"dw %s" % spill_symbol_name)
thisptr = table[table_idx]
if table_idx < len(table) - 1:
#fixup the next pointer in the table
table[table_idx + 1] = thisptr + 4
overflow[spill_symbol_name] = cur_string
print("Spilled 0x{0:x} bytes to overflow bank".format(
abs(string_bytes_saved)))
#Actually generate our section source code now.
srcdata = format_section("String table %s" % (bank["symbol"], ),
flat(bank["basebank"], bank["baseaddr"]))
srcdata += format_symbol(bank["symbol"], True)
for table_ptr in table:
srcdata += " dw $%x\n" % (table_ptr)
for packed_string in packed_strings:
srcdata += format_directives(packed_string)
return (srcdata, overflow)