def extract(args):
with open(args.filenames[0], 'rb') as rom:
asmsrc, files = rip_msprite_mtable(rom, args.metatable_loc)
for filename, data in files.items():
install_path(os.path.dirname(filename))
with io.open(filename, "w+", encoding="utf-8") as spr_csv:
spr_csv.write(data)
print(asmsrc)
def extract_bank(args, rom, bank, bank_dir):
this_bank, this_table = decompress_bank(rom, 0x4000 * bank)
for j, data in enumerate(this_bank):
csvpath = os.path.join(args.output, "unknown", bank_dir)
install_path(csvpath)
with open(os.path.join(csvpath, "{0:x}.csv".format(j)),
"w+") as csvout:
csvwriter = csv.writer(csvout)
for row in data[:-1]:
csvwriter.writerow(row)
#Workaround for a csv.writer bug
if len(data) > 0 and len(data[-1]) == 0:
csvout.write(",")
elif len(data) > 0:
csvwriter.writerow(data[-1])
def wikisync(args):
charmap = parse_charmap(args.charmap)
banknames = parse_bank_names(args.banknames)
for h, bank in enumerate(banknames):
api_url = "http://wiki.telefang.net/api.php?action=query&titles=Wikifang:Telefang_1_Translation_Patch/Text_dump/{}&format=json&prop=revisions&rvprop=content".format(
bank["wikiname"].strip())
full_wikiname = "Wikifang:Telefang 1 Translation Patch/Text dump/{}".format(
bank["wikiname"].strip())
wikifile = urllib.request.urlopen(api_url)
data = json.load(wikifile)
for pageid in data["query"]["pages"]:
if data["query"]["pages"][pageid]["title"] == full_wikiname:
wikidir = os.path.join(args.input, bank["basedir"])
wikipath = os.path.join(args.input, bank["filename"])
install_path(wikidir)
with open(wikipath, "w", encoding="utf-8") as bank_wikitext:
bank_wikitext.write(
data["query"]["pages"][pageid]["revisions"][0]["*"])
def extract(args):
charmap = parse_charmap(args.charmap)
tablenames = parse_tablenames(args.tablenames)
with open(args.rom, 'rb') as rom:
#Extract a list of pointers each index is expecting
#This is used for trash byte detection later
for table in tablenames:
if table["format"] != "index":
continue
try:
all_ptrs = tablenames[table["foreign_id"]]["expected_ptrs"]
except KeyError:
all_ptrs = []
rom.seek(flat(table["basebank"], table["baseaddr"]))
for i in range(table["count"]):
ptr = PTR.unpack(rom.read(2))[0]
addr = flat(table["basebank"], ptr)
if addr not in all_ptrs:
all_ptrs.append(addr)
all_ptrs.sort()
tablenames[table["foreign_id"]]["expected_ptrs"] = all_ptrs
for table in tablenames:
#Indexes are extracted in a second pass
if table["format"] == "index":
continue
entries = []
reverse_entries = {}
try:
expected_ptrs = table["expected_ptrs"]
except KeyError:
expected_ptrs = None
csvdir = os.path.join(args.input, table["basedir"])
csvpath = os.path.join(args.input, table["filename"])
install_path(csvdir)
with open(csvpath, "w+", encoding="utf-8") as table_csvfile:
csvwriter = csv.writer(table_csvfile)
csvwriter.writerow(["#", args.language])
if table["format"] == "table":
for i in range(table["count"]):
rom.seek(
flat(table["basebank"],
table["baseaddr"] + i * table["stride"]))
reverse_entries[rom.tell()] = len(entries)
entries.append(rom.tell())
data = extract_string(rom, charmap, table["stride"],
expected_ptrs).encode("utf-8")
idx = "{0}".format(i + 1).encode("utf-8")
csvwriter.writerow([idx, data])
elif table["format"] == "block":
rom.seek(flat(table["basebank"], table["baseaddr"]))
for i in range(table["count"]):
reverse_entries[rom.tell()] = len(entries)
entries.append(rom.tell())
data = extract_string(rom, charmap, None,
expected_ptrs).encode("utf-8")
idx = "{0}".format(i + 1).encode("utf-8")
csvwriter.writerow([idx, data])
#Save these for later
table["entries"] = entries
table["reverse_entries"] = reverse_entries
#OK, now we can extract the indexes
for table in tablenames:
if table["format"] != "index":
continue
foreign_ptrs = tablenames[table["foreign_id"]]["reverse_entries"]
rom.seek(flat(table["basebank"], table["baseaddr"]))
csvdir = os.path.join(args.input, table["basedir"])
csvpath = os.path.join(args.input, table["filename"])
install_path(csvdir)
with open(csvpath, "w+", encoding="utf-8") as table_csvfile:
csvwriter = csv.writer(table_csvfile)
pretty_row_length = math.ceil(math.sqrt(table["count"]))
cur_row = []
for i in range(table["count"]):
ptr = PTR.unpack(rom.read(2))[0]
addr = flat(table["basebank"], ptr)
cur_row.append("{0}".format(foreign_ptrs[addr] +
1).encode("utf-8"))
if len(cur_row) >= pretty_row_length:
csvwriter.writerow(cur_row)
cur_row = []
if len(cur_row) > 0:
csvwriter.writerow(cur_row)
def extract(args):
charmap = parse_charmap(args.charmap)
banknames = parse_bank_names(args.banknames)
banknames = extract_metatable_from_rom(args.rom, charmap, banknames, args)
with open(args.rom, 'rb') as rom:
for bank in banknames:
wikitext = ["{|", "|-", "!Pointer", "!" + args.language]
csvdata = [["Pointer", args.language]]
rom.seek(flat(bank["basebank"], bank["baseaddr"]))
addr = bank["baseaddr"]
end = 0x8000
#Autodetect the end/length of the table by finding the lowest
#pointer that isn't stored after an existing pointer
while addr < end:
next_ptr = PTR.unpack(rom.read(2))[0]
#Reject obviously invalid pointers
if (next_ptr < addr or next_ptr > 0x7FFF):
break
end = min(end, next_ptr)
addr += 2
tbl_length = (addr - bank["baseaddr"]) // 2
#Actually extract our strings
string = []
#Stores the actual end of the last string, used for alias detection
last_start = 0xFFFF
last_end = 0xFFFF
last_nonaliasing_row = -1
#Also store if a redirected/overflowed row is being extracted
redirected = False
old_loc = None
for i in range(tbl_length):
csvrow = [
"0x{0:x}".format(
flat(bank["basebank"], bank["baseaddr"] + i * 2))
]
wikitext.append("|-")
wikitext.append("|0x{0:x}".format(
flat(bank["basebank"], bank["baseaddr"] + i * 2)))
rom.seek(flat(bank["basebank"], bank["baseaddr"] + i * 2))
read_ptr = PTR.unpack(rom.read(2))[0]
#Attempt to autodetect "holes" in the text data.
next_ptr = PTR.unpack(rom.read(2))[0]
expected_length = next_ptr - read_ptr
if i >= tbl_length - 1:
expected_length = -1 #maximum length by far
#Two different alias detects:
#First, we try to see if this pointer matches another pointer
#in the table.
rom.seek(flat(bank["basebank"], bank["baseaddr"]))
for j in range(i):
if read_ptr == PTR.unpack(rom.read(2))[0]:
#Aliased pointer!
csvrow.append("<ALIAS ROW 0x{0:x}>".format(j))
wikitext.append("|«ALIAS ROW 0x{0:x}»".format(j))
print(
"Pointer at 0x{0:x} fully aliases pointer 0x{1:x}".
format(
flat(bank["basebank"],
bank["baseaddr"] + i * 2),
flat(bank["basebank"],
bank["baseaddr"] + j * 2)))
break
else:
#Second, we try to see if this pointer is in the middle of
#the last string.
#This alias detection breaks when the previous row uses the
#overflow code, so disable it if so.
if i > 0 and read_ptr < last_end - 1 and not redirected:
print(
"Pointer at 0x{0:x} partially aliases previous pointer"
.format(rom.tell() - 2))
csvrow.append(
"<ALIAS ROW 0x{0:x} INTO 0x{1:x}>".format(
last_nonaliasing_row, read_ptr - last_start))
wikitext.append(
"|«ALIAS ROW 0x{0:x} INTO 0x{1:x}»".format(
last_nonaliasing_row, read_ptr - last_start))
continue
read_length = 1
first_read = True
rom.seek(flat(bank["basebank"], read_ptr))
#Now we can initialize these...
redirected = False
old_loc = None
while (rom.tell() % 0x4000 < 0x3FFF or rom.tell() == flat(
bank["basebank"], bank["baseaddr"])):
next_chara = CHARA.unpack(rom.read(1))[0]
while (rom.tell() % 0x4000 < 0x3FFF or rom.tell() ==
flat(bank["basebank"], bank["baseaddr"])) and (
read_length <= expected_length or first_read
or redirected
) and next_chara != 0xE0: #E0 is end-of-string
if next_chara < 0xE0 and next_chara in charmap[
1]: #Control codes are the E0 block
string.append(charmap[1][next_chara])
elif next_chara in reverse_specials and specials[
reverse_specials[next_chara]].redirect:
#Redirecting opcodes are transparently removed from the extracted text.
this_special = specials[
reverse_specials[next_chara]]
if this_special.bts:
read_length += this_special.bts
fmt = "<" + ("", "B",
"H")[this_special.bts]
word = struct.unpack(
fmt, rom.read(this_special.bts))[0]
if word < 0x4000 or word > 0x7FFF:
#Overflowing into RAM is illegal - use the jump opcode.
#Overflowing into ROM0 is technically not illegal, but
#unorthodox enough that we're going to disallow it.
string.append(
format_literal(this_special.byte))
string.append(
format_literal(
word & 0xFF, charmap[1]))
string.append(
format_literal(
word >> 8, charmap[1]))
else:
#We need to do this right now to avoid breaking hole detection
old_loc = rom.tell()
read_length = rom.tell() - flat(
bank["basebank"], read_ptr)
rom.seek(flat(args.overflow_bank,
word))
redirected = True
else:
raise RuntimeError(
"Invalid specials dictionary. Redirecting special character is missing bts."
)
elif next_chara in reverse_specials:
#This must be the work of an 「ＥＮＥＭＹ ＳＴＡＮＤ」
this_special = specials[
reverse_specials[next_chara]]
if this_special.bts:
read_length += this_special.bts
fmt = "<" + ("", "B",
"H")[this_special.bts]
word = struct.unpack(
fmt, rom.read(this_special.bts))[0]
string.append(
format_control_code(
reverse_specials[next_chara],
word))
else:
string.append(
format_control_code(
reverse_specials[next_chara]))
if this_special.end:
first_read = False
break
#elif next_chara == 0xE2:
#Literal newline
# string.append(u"\n")
else:
#Literal specials
string.append(format_literal(next_chara))
next_chara = CHARA.unpack(rom.read(1))[0]
#Explicitly stop updating read_length if the
#overflow opcode is used. Otherwise we'd think we
#read thousands or negative thousands of chars
if not redirected:
read_length = rom.tell() - flat(
bank["basebank"], read_ptr)
#After the main extraction loop
if read_length >= expected_length:
break
else:
#Detect nulls (spaces) after the end of a string
#and append them to avoid creating a new pointer row
loc = rom.tell()
if redirected:
loc = old_loc
while CHARA.unpack(rom.read(1))[0] == charmap[0][
" "] and read_length < expected_length:
string.append(" ")
loc += 1
read_length += 1
rom.seek(loc) #cleanup
if read_length >= expected_length:
break
else:
#There's a hole in the ROM!
#Disassemble the next string.
print("Inaccessible data found at 0x{0:x}".
format(flat(bank["basebank"], read_ptr)))
csvrow.append("".join(string))
wikitext.append("|" + "".join(string))
string = []
csvdata.append(csvrow)
csvrow = ["(No pointer)"]
wikitext.append("|-")
wikitext.append("|(No pointer)")
read_length += 1
csvrow.append("".join(string))
wikitext.append("|" + "".join(string))
string = []
#Store the actual end pointer for later use.
last_start = read_ptr
last_end = read_ptr + read_length
last_nonaliasing_row = i
csvdata.append(csvrow)
wikitext.append("|-")
wikitext.append("|}")
wikitext = "\n".join(wikitext)
wikidir = os.path.join(args.input, bank["basedir"])
wikipath = os.path.join(args.input, bank["legacy_filename"])
csvpath = os.path.join(args.input, bank["filename"])
install_path(wikidir)
#with open(wikipath, "w+", encoding="utf-8") as bank_wikitext:
#bank_wikitext.write(wikitext)
with open(csvpath, "w+", encoding="utf-8") as bank_csvtext:
csvwriter = csv.writer(bank_csvtext)
for csvrow in csvdata:
csvwriter.writerow(csvrow)