def asm(args): """Generate the ASM for string tables. This operation needs to be performed once, plus Generated ASM will be printed to console. This portion of the script is intended to be piped into a file.""" charmap = mainscript_text.parse_charmap(args.charmap) tablenames = parse_tablenames(args.tablenames) for table in tablenames: print('SECTION "' + table["symbol"] + ' Section", ' + mainscript_text.format_sectionaddr_rom( mainscript_text.flat(table["basebank"], table["baseaddr"]))) print(table["symbol"] + '::') print('\tINCBIN "' + os.path.join(args.output, table["objname"]).replace("\\", "/") + '"') print(table["symbol"] + '_END') print('')
def rip_msprite_mtable(rom, offset=0x094D, count=9): """Rip an entire ROM's metasprite data.""" cloc = rom.tell() rom.seek(offset) #The metasprite metatable is oddly organized here. #Banks in one place, pointers in the other. banks = [] ptrs = [] for i in range(0, count): banks.append(CHARA.unpack(rom.read(1))[0]) for i in range(0, count): ptrs.append(PTR.unpack(rom.read(2))[0]) asmsrc = "SECTION \"MetaSprite metatable\", " + format_sectionaddr_rom( offset) + "\n" asmsrc += "MetaspriteBankMetatable::\n" for bank in banks: asmsrc += " db BANK(MetaSprite_" + "{0:x}".format(bank) + ")\n" asmsrc += "MetaspriteAddressMetatable::\n" for bank, ptr in zip(banks, ptrs): asmsrc += " dw MetaSprite_" + "{0:x}".format(bank) + "\n" asmsrc += "\n" files = {} for bank, ptr in zip(banks, ptrs): table_asmsrc, table_files = rip_msprite_table(rom, flat(bank, ptr)) asmsrc += table_asmsrc + "\n" files.update(table_files) return (asmsrc, files)
def decompress_bank(rom, offset=None): """Decompress an entire table of compressed data. First return value is an array of data, individual data items match the return value of decompress_tilemap. Data is ordered in the same order that the ROM orders it. Second return value is the extracted table, represented as indexes into the data array. If offset is given, ROM will be read from that position. Your existing ROM position will be preserved.""" if offset is None: offset = rom.tell() last = rom.tell() rom.seek(offset) ret_dat = [] order_dat = [] ptr_table = [] #List of PTRs as specified in the table ptr_data = [] #List of PTRs containing data, some unused ptr_data_org = {} #Decompressed data ptr_data_index = {} #The table consists of multiple pointers followed immediately by compressed #tilemap data. There is no explicit length, so we discover the length of #the pointer table by reading and decompressing pointers until we overrun #already-decompressed data. Oh, and they don't come in a specified order #either. first_data_ptr = 0xFFFF last_data_ptr = 0x0000 last_ptr = None start_ptr, this_bank = mainscript_text.banked(offset) #Extract the table first. while mainscript_text.banked(rom.tell())[0] < first_data_ptr: this_ptr = mainscript_text.PTR.unpack(rom.read(2))[0] ptr_table.append(this_ptr) if this_ptr <= first_data_ptr: first_data_ptr = this_ptr if last_data_ptr <= this_ptr: last_data_ptr = this_ptr #Extract the data second this_ptr = first_data_ptr while this_ptr <= last_data_ptr: decomp_data, decomp_len = decompress_tilemap( rom, mainscript_text.flat(this_bank, this_ptr)) ptr_data.append(this_ptr) ptr_data_org[this_ptr] = decomp_data this_ptr += decomp_len #Resolve PTRs into numerical indexes and return them. for data_index, data_ptr in enumerate(ptr_data): ret_dat.append(ptr_data_org[data_ptr]) ptr_data_index[data_ptr] = data_index for table_index, table_ptr in enumerate(ptr_table): order_dat.append(ptr_data_index[table_ptr]) rom.seek(last) return ret_dat, order_dat
def make_tbl(args): charmap = mainscript_text.parse_charmap(args.charmap) tablenames = parse_tablenames(args.tablenames) for table in tablenames: #Indexes are compiled in a second pass if table["format"] == "index": continue #If filenames are specified, only export banks that are mentioned there if len(args.filenames) > 0 and table["objname"] not in args.filenames: continue print("Compiling " + table["filename"]) #Open and parse the data with open(os.path.join(args.input, table["filename"]), "r", encoding="utf-8") as csvfile: csvreader = csv.reader(csvfile) headers = None rows = [] for row in csvreader: if headers is None: headers = row else: rows.append(row) #Determine what column we want index_col = headers.index("#") try: str_col = headers.index(args.language) except ValueError: str_col = index_col #Pack our strings packed_strings = [] baseaddr = table["baseaddr"] entries = [] reverse_entries = {} for i, row in enumerate(rows): if str_col >= len(row): print("WARNING: ROW {} IS MISSING IT'S TEXT!!!".format(i)) packed_strings.append(b"") continue packed = mainscript_text.pack_text(row[str_col], specials, charmap[0], None, args.window_width, 1, memory_widths, True) if "stride" in table: if len(packed) > table["stride"]: print("WARNING: Row {} is too long for the current string table stride of {} in table {}.".format(i, table["stride"], table["filename"])) packed = packed[0:table["stride"]] else: #Pad the string out with E0s. packed = packed + bytes([0xE0] * (table["stride"] - len(packed))) elif b"\xe0" not in packed: #Any data beyond an E0 is understood to be trash bytes; thus, #it does not recieve the implicit terminator. packed = packed + b"\xe0" packed_strings.append(packed) reverse_entries[mainscript_text.flat(table["basebank"], baseaddr)] = len(entries) entries.append(mainscript_text.flat(table["basebank"], baseaddr)) baseaddr += len(packed) #Save these for later table["entries"] = entries table["reverse_entries"] = reverse_entries #Write the data out to the object files. We're done here! if not os.path.exists(os.path.dirname(os.path.join(args.output, table["objname"]))): os.makedirs(os.path.dirname(os.path.join(args.output, table["objname"]))) with open(os.path.join(args.output, table["objname"]), "wb") as objfile: for line in packed_strings: objfile.write(line) for table in tablenames: #Now's the time to compile indexes if table["format"] != "index": continue #If filenames are specified, only export banks that are mentioned there if len(args.filenames) > 0 and table["objname"] not in args.filenames: continue print("Compiling " + table["filename"]) foreign_ptrs = tablenames[table["foreign_id"]]["entries"] packed_strings = [] entries = [] reverse_entries = {} #Open and parse the data with open(os.path.join(args.input, table["filename"]), "r", encoding="utf-8") as csvfile: csvreader = csv.reader(csvfile) for row in csvreader: for cell in row: packed_strings.append(mainscript_text.PTR.pack(foreign_ptrs[int(cell, 10) - 1] % 0x4000 + 0x4000)) #Write the data out to the object files. We're done here! with open(os.path.join(args.output, table["objname"]), "wb") as objfile: for line in packed_strings: objfile.write(line)
def extract(args): charmap = mainscript_text.parse_charmap(args.charmap) tablenames = parse_tablenames(args.tablenames) with open(args.rom, 'rb') as rom: #Extract a list of pointers each index is expecting #This is used for trash byte detection later for table in tablenames: if table["format"] != "index": continue try: all_ptrs = tablenames[table["foreign_id"]]["expected_ptrs"] except KeyError: all_ptrs = [] rom.seek(mainscript_text.flat(table["basebank"], table["baseaddr"])) for i in range(table["count"]): ptr = mainscript_text.PTR.unpack(rom.read(2))[0] addr = mainscript_text.flat(table["basebank"], ptr) if addr not in all_ptrs: all_ptrs.append(addr) all_ptrs.sort() tablenames[table["foreign_id"]]["expected_ptrs"] = all_ptrs for table in tablenames: #Indexes are extracted in a second pass if table["format"] == "index": continue entries = [] reverse_entries = {} try: expected_ptrs = table["expected_ptrs"] except KeyError: expected_ptrs = None csvdir = os.path.join(args.input, table["basedir"]) csvpath = os.path.join(args.input, table["filename"]) mainscript_text.install_path(csvdir) with open(csvpath, "w+", encoding="utf-8") as table_csvfile: csvwriter = csv.writer(table_csvfile) csvwriter.writerow(["#", args.language]) if table["format"] == "table": for i in range(table["count"]): rom.seek(mainscript_text.flat(table["basebank"], table["baseaddr"] + i * table["stride"])) reverse_entries[rom.tell()] = len(entries) entries.append(rom.tell()) data = extract_string(rom, charmap, table["stride"], expected_ptrs).encode("utf-8") idx = "{0}".format(i + 1).encode("utf-8") csvwriter.writerow([idx, data]) elif table["format"] == "block": rom.seek(mainscript_text.flat(table["basebank"], table["baseaddr"])) for i in range(table["count"]): reverse_entries[rom.tell()] = len(entries) entries.append(rom.tell()) data = extract_string(rom, charmap, None, expected_ptrs).encode("utf-8") idx = "{0}".format(i + 1).encode("utf-8") csvwriter.writerow([idx, data]) #Save these for later table["entries"] = entries table["reverse_entries"] = reverse_entries #OK, now we can extract the indexes for table in tablenames: if table["format"] != "index": continue foreign_ptrs = tablenames[table["foreign_id"]]["reverse_entries"] rom.seek(mainscript_text.flat(table["basebank"], table["baseaddr"])) csvdir = os.path.join(args.input, table["basedir"]) csvpath = os.path.join(args.input, table["filename"]) mainscript_text.install_path(csvdir) with open(csvpath, "w+", encoding="utf-8") as table_csvfile: csvwriter = csv.writer(table_csvfile) pretty_row_length = math.ceil(math.sqrt(table["count"])) cur_row = [] for i in range(table["count"]): ptr = mainscript_text.PTR.unpack(rom.read(2))[0] addr = mainscript_text.flat(table["basebank"], ptr) cur_row.append("{0}".format(foreign_ptrs[addr] + 1).encode("utf-8")) if len(cur_row) >= pretty_row_length: csvwriter.writerow(cur_row) cur_row = [] if len(cur_row) > 0: csvwriter.writerow(cur_row)
def rip_msprite_table(rom, offset=None): """Given a ROM and an offset, rip a metatable of metasprites. Ripped data will be returned as an array of ASM and a dict listing files to save.""" cloc = rom.tell() if offset is not None: rom.seek(offset) else: offset = cloc table_ptr, bank = banked(offset) ptrs = [] end = 0x7FFF cur = table_ptr table_ptr_list = [] ripped_sprites = {} while cur < end: offset_ptr = PTR.unpack(rom.read(2))[0] cur += 2 end = min(offset_ptr, end) if offset_ptr not in list(ripped_sprites.keys()): try: ripped_sprites[offset_ptr] = rip_msprite( rom, flat(bank, offset_ptr)) except KeyError: #If a metasprite points at undecipherable data, assume it's a #dummy pointer and don't generate a metasprite for it, since #it's clearly run off the end of the table. ripped_sprites[offset_ptr] = [] table_ptr_list.append(offset_ptr) #Now that we have the tables and their data, generate the ASM for it... data_asmsrc = "" table_asmsrc = "" files = {} table_symbol = "MetaSprite_" + "{0:x}".format(bank) table_asmsrc += "SECTION \"" + table_symbol + "\", ROMX[$" + "{0:x}".format( table_ptr) + "], BANK[$" + "{0:x}".format(bank) + "]\n" table_asmsrc += table_symbol + "::\n" sorted_ptrs = list(ripped_sprites.keys()) sorted_ptrs.sort() last_ptrkey_end = 0x0000 for ptrkey in sorted_ptrs: val = ripped_sprites[ptrkey] symbol = table_symbol + "_" + "{0:x}".format(ptrkey) file = "gfx/unknown/metasprite_" + "{0:x}".format( bank) + "/" + "{0:x}".format(ptrkey) + ".sprite.csv" binfile = "gfx/unknown/metasprite_" + "{0:x}".format( bank) + "/" + "{0:x}".format(ptrkey) + ".sprite.bin" if last_ptrkey_end != ptrkey: data_asmsrc += "SECTION \"" + symbol + "\", ROMX[$" + "{0:x}".format( ptrkey) + "], BANK[$" + "{0:x}".format(bank) + "]\n" data_asmsrc += symbol + "::\n" #ASSUMPTION: All 0 size metasprites are dummy pointers if len(val) > 0: data_asmsrc += " INCBIN \"" + binfile + "\"\n" data_asmsrc += symbol + "_END::\n" files[file] = "X,Y,Tile Offset,Attribute Mixing,Attributes\n" for spritecfg in val: files[file] += "{0:x},{1:x},{2:x},{3},{4:x}\n".format( spritecfg['x'], spritecfg['y'], spritecfg['tile'], spritecfg['attrib_mode'], spritecfg['attribs']) last_ptrkey_end = ptrkey + 1 + len(val) * 5 for ptr in table_ptr_list: symbol = table_symbol + "_" + "{0:x}".format(ptr) table_asmsrc += " dw " + symbol + "\n" asm_src = table_asmsrc + "\n" + data_asmsrc rom.seek(cloc) return (asm_src, files)
def make_tbl(args): charmap = mainscript_text.parse_charmap(args.charmap) tablenames = parse_tablenames(args.tablenames) for table in tablenames: #Indexes are compiled in a second pass if table["format"] == "index": continue #If filenames are specified, only export banks that are mentioned there if len(args.filenames) > 0 and table["objname"] not in args.filenames: continue print("Compiling " + table["filename"]) #Open and parse the data with open(os.path.join(args.input, table["filename"]), "r", encoding="utf-8") as csvfile: csvreader = csv.reader(csvfile) headers = None rows = [] for row in csvreader: if headers is None: headers = row else: rows.append(row) #Determine what column we want index_col = headers.index("#") try: str_col = headers.index(args.language) except ValueError: str_col = index_col #Pack our strings packed_strings = [] baseaddr = table["baseaddr"] entries = [] reverse_entries = {} for i, row in enumerate(rows): if str_col >= len(row): print("WARNING: ROW {} IS MISSING IT'S TEXT!!!".format(i)) packed_strings.append(b"") continue packed = mainscript_text.pack_text(row[str_col], specials, charmap[0], None, args.window_width, 1, memory_widths, True) if "stride" in table: if len(packed) > table["stride"]: print("WARNING: Row {} is too long for the current string table stride of {} in table {}.".format(i, table["stride"], table["filename"])) packed = packed[0:table["stride"]] else: #Pad the string out with E0s. packed = packed + bytes([0xE0] * (table["stride"] - len(packed))) elif b"\xe0" not in packed: #Any data beyond an E0 is understood to be trash bytes; thus, #it does not recieve the implicit terminator. packed = packed + b"\xe0" packed_strings.append(packed) reverse_entries[mainscript_text.flat(table["basebank"], baseaddr)] = len(entries) entries.append(mainscript_text.flat(table["basebank"], baseaddr)) baseaddr += len(packed) #Save these for later table["entries"] = entries table["reverse_entries"] = reverse_entries #Write the data out to the object files. We're done here! if not os.path.exists(os.path.dirname(os.path.join(args.output, table["objname"]))): try: os.makedirs(os.path.dirname(os.path.join(args.output, table["objname"]))) except FileExistsError: pass with open(os.path.join(args.output, table["objname"]), "wb") as objfile: for line in packed_strings: objfile.write(line) for table in tablenames: #Now's the time to compile indexes if table["format"] != "index": continue #If filenames are specified, only export banks that are mentioned there if len(args.filenames) > 0 and table["objname"] not in args.filenames: continue print("Compiling " + table["filename"]) foreign_ptrs = tablenames[table["foreign_id"]]["entries"] packed_strings = [] entries = [] reverse_entries = {} #Open and parse the data with open(os.path.join(args.input, table["filename"]), "r", encoding="utf-8") as csvfile: csvreader = csv.reader(csvfile) for row in csvreader: for cell in row: packed_strings.append(mainscript_text.PTR.pack(foreign_ptrs[int(cell, 10) - 1] % 0x4000 + 0x4000)) #Write the data out to the object files. We're done here! with open(os.path.join(args.output, table["objname"]), "wb") as objfile: for line in packed_strings: objfile.write(line)
def asm(args): """Generate the ASM for string tables. This operation needs to be performed once, plus Generated ASM will be printed to console. This portion of the script is intended to be piped into a file.""" charmap = mainscript_text.parse_charmap(args.charmap) tablenames = parse_tablenames(args.tablenames) for table in tablenames: print('SECTION "' + table["symbol"] + ' Section", ' + mainscript_text.format_sectionaddr_rom(mainscript_text.flat(table["basebank"], table["baseaddr"]))) print(table["symbol"] + '::') print('\tINCBIN "' + os.path.join(args.output, table["objname"]).replace("\\", "/") + '"') print(table["symbol"] + '_END') print('')
def decompress_bank(rom, offset=None): """Decompress an entire table of compressed data. First return value is an array of data, individual data items match the return value of decompress_tilemap. Data is ordered in the same order that the ROM orders it. Second return value is the extracted table, represented as indexes into the data array. If offset is given, ROM will be read from that position. Your existing ROM position will be preserved.""" if offset is None: offset = rom.tell() last = rom.tell() rom.seek(offset) ret_dat = [] order_dat = [] ptr_table = [] #List of PTRs as specified in the table ptr_data = [] #List of PTRs containing data, some unused ptr_data_org = {} #Decompressed data ptr_data_index = {} #The table consists of multiple pointers followed immediately by compressed #tilemap data. There is no explicit length, so we discover the length of #the pointer table by reading and decompressing pointers until we overrun #already-decompressed data. Oh, and they don't come in a specified order #either. first_data_ptr = 0xFFFF last_data_ptr = 0x0000 last_ptr = None start_ptr, this_bank = mainscript_text.banked(offset) #Extract the table first. while mainscript_text.banked(rom.tell())[0] < first_data_ptr: this_ptr = mainscript_text.PTR.unpack(rom.read(2))[0] ptr_table.append(this_ptr) if this_ptr <= first_data_ptr: first_data_ptr = this_ptr if last_data_ptr <= this_ptr: last_data_ptr = this_ptr #Extract the data second this_ptr = first_data_ptr while this_ptr <= last_data_ptr: decomp_data, decomp_len = decompress_tilemap(rom, mainscript_text.flat(this_bank, this_ptr)) ptr_data.append(this_ptr) ptr_data_org[this_ptr] = decomp_data this_ptr += decomp_len #Resolve PTRs into numerical indexes and return them. for data_index, data_ptr in enumerate(ptr_data): ret_dat.append(ptr_data_org[data_ptr]) ptr_data_index[data_ptr] = data_index for table_index, table_ptr in enumerate(ptr_table): order_dat.append(ptr_data_index[table_ptr]) rom.seek(last) return ret_dat, order_dat