def extract_scripts(self, filename, output_folder, debug=False): self.offsets = [] self.scripts = [] self.functions = [] self.func_map = {} # {offset: [func, count]} output = "" print(filename) fin = open(os.path.join(self.config.path, filename), "rb+") fin = (BinaryIO.reader(fin.read())).adapter(fin) fin.seek(0) fin.seek(0, os.SEEK_END) filesize = fin.tell() offset = 0 num = 0 song_names = [''] export_sounds(fin, 0x0, song_names, filename, output_folder, 'Script') return (output, offset)
def load(self, reader): reader = BinaryIO.reader(reader) start = reader.tell() self.magic = reader.read(4) size = reader.readUInt32() self.texinfo.load(reader) self.tex4x4info.load(reader) self.palinfo.load(reader) # Build dicts reader.seek(start + self.texinfo._lookupofs) self.texdict.load(reader) self.texparams = [] for i in xrange(self.texdict.num): imgParam, extra = struct.unpack('II', self.texdict.data[i]) self.texparams.append( TexParam((imgParam & 0xFFFF) << 3, 8 << ((imgParam >> 20) & 0x7), 8 << ((imgParam >> 23) & 0x7), (imgParam >> 26) & 0x7, (imgParam >> 29) & 0x1)) reader.seek(start + self.palinfo._lookupofs) self.paldict.load(reader) self.palparams = [] for i in xrange(self.paldict.num): offset, flag = struct.unpack('HH', self.paldict.data[i]) self.palparams.append(PalParam(offset << 3, flag)) # Read data. reader.seek(start + self.texinfo._dataofs) self.texdata = reader.read(self.texinfo._datasize) reader.seek(start + self.palinfo._dataofs) self.paldata = reader.read(self.palinfo._datasize) # TODO 4x4 if size: reader.seek(start + size) self._images = None
def init(self, filename, base_address): output = "" self.filename = filename self.file = open(filename, "rb+") self.file = (BinaryIO.reader(self.file.read())).adapter(self.file) self.file.seek(0) self.base_address = base_address
def __init__(self, reader, compression=COMPRESSION_LZ77): handle = BinaryIO.reader(reader) start = handle.tell() data = array.array('B', handle.read()) self.header = LZHeader._make([compression, len(data)]) handle.truncate(start) handle.seek(start) handle.write(pack('I', self.header.flag | (self.header.size << 8))) out = array.array('B') out.append(0) out.extend(data[:4]) pos = 4 endpos = len(data) search_start = 0 max_len = 0x12 control_bit = 4 flag = 0 flag_pos = 0 while pos < endpos: best_pos = None best_len = 0 for check in xrange(search_start, pos): for idx in xrange(0, min(max_len + 1, pos - check, endpos - pos)): if data[check + idx] != data[pos + idx]: break if idx >= best_len: best_len = idx best_pos = check if idx >= max_len: break control_bit -= 1 if best_len < 3: out.append(data[pos]) pos += 1 else: flag |= 1 << control_bit head = (best_len - 3) << 0xC head |= pos - best_pos - 1 out.append(head >> 8) out.append(head & 0xFF) pos += best_len if control_bit <= 0: if pos >= 0x400: search_start = pos - 0x400 control_bit = 8 out[flag_pos] = flag flag_pos = len(out) flag = 0 out.append(0) out[flag_pos] = flag handle.write(out.tostring()) self.handle = handle
def add_binary(self, handle, name='.text', address=0x02000000): section = Section(name, SectionHeader.TYPE_PROGBITS) reader = BinaryIO.reader(handle) with reader.seek(0): section.data.write(reader.read()) section.header.address = address section.header.flags = section.header.FLAG_EXECINSTR \ | section.header.FLAG_ALLOC self.add_section(section) symbol = self.add_symbol(name, address, Symbol.TYPE_SECTION) symbol.type_ = symbol.TYPE_SECTION
def extract_scripts(filename, output_folder): output = "" print(filename) fin = open(os.path.join(configuration.Config().path, filename), "rb+") fin = (BinaryIO.reader(fin.read())).adapter(fin) output = dumb_scripts(fin, 'Script') head, tail = os.path.split(filename) with open(os.path.join(output_folder + tail.replace(".bin", ".s")), 'w') as out: out.write(output) return output
def load(self, reader): reader = BinaryIO.reader(reader) start = reader.tell() self.magic = reader.read(4) self.endian = reader.readUInt16() self.version = reader.readUInt16() size = reader.readUInt32() headersize = reader.readUInt16() numblocks = reader.readUInt16() if headersize: reader.seek(start + headersize) self.fatb.load(reader) self.fntb.load(reader) self.fimg.load(reader) if size: reader.seek(start + size)
def __init__(self, reader): handle = BinaryIO.reader(reader) start = handle.tell() raw_header = unpack('I', handle.read(4))[0] self.header = LZHeader._make([raw_header & 0xFF, raw_header >> 8]) if self.header.flag == 0x11: lz_ss = True elif self.header.flag == 0x10: lz_ss = False else: raise ValueError('Invalid compression flag: {0}'.format( self.header.flag)) out = [] while len(out) < self.header.size: flag = unpack('B', handle.read(1))[0] for x in range(7, -1, -1): if len(out) >= self.header.size: break if not (flag >> x) & 0x1: r = handle.read(1) out.append(r) continue head, = unpack('>H', handle.read(2)) if not lz_ss: count = ((head >> 12) & 0xF) + 3 back = head & 0xFFF else: ind = (head >> 12) & 0xF if not ind: tail = unpack('B', handle.read(1))[0] count = (head >> 4) + 0x11 back = ((head & 0xF) << 8) | tail elif ind == 1: tail = unpack('>H', handle.read(2))[0] count = (((head & 0xFFF) << 4) | (tail >> 12)) + 0x111 back = tail & 0xFFF else: count = ind + 1 back = head & 0xFFF cursz = len(out) for y in range(count): out.append(out[cursz - back - 1 + y]) self.data = "".join(out) handle.seek(start) handle.write(self.data) handle.seek(start) self.handle = handle
def load(self, reader): reader = BinaryIO.reader(reader) start = reader.tell() self.magic = reader.read(4) headersize = reader.readUInt32() u1 = reader.readUInt32() u2 = reader.readUInt32() u3 = reader.readUInt32() size = reader.readUInt32() u4 = reader.readUInt32() if headersize: reader.seek(start + headersize) self.fato.load(reader) self.fatb.load(reader) self.fimb.load(reader) if size: reader.seek(start + size)
def load(self, reader): reader = BinaryIO.reader(reader) self.conditions = [] self.__getitem__ = self.conditions.__getitem__ self.__setitem__ = self.conditions.__setitem__ self.u0 = reader.readUInt16() self.u2 = reader.readUInt16() try: self.u4 = reader.readUInt16() except: self.u4 = 0 return while True: try: cond = ScriptCondition() cond.load(reader) self.conditions.append(cond) except: break
def decompress_arm9(game): """Creates an arm9.dec.bin in the Game's workspace This file will be created even if arm9.bin is already decompressed """ workspace = game.files.directory try: if os.path.getsize(os.path.join(workspace, 'arm9.dec.bin')) > 0: return except OSError: pass with open(os.path.join(workspace, 'header.bin')) as header: header.seek(0x24) entry, ram_offset, size = struct.unpack('III', header.read(12)) with open(os.path.join(workspace, 'arm9.bin')) as arm9,\ open(os.path.join(workspace, 'arm9.dec.bin'), 'w') as arm9dec: arm9.seek(game.load_info - ram_offset + 0x14) end, u18, beacon, unbeacon = struct.unpack('IIII', arm9.read(16)) assert beacon & 0xFFFF0000 == ARM9_BLZ_BEACON & 0xFFFF0000 assert unbeacon & 0xFFFF == ARM9_BLZ_UNBEACON & 0xFFFF # TODO: if beacons do not match, scan ARM9 for beacon try: assert end arm9.seek(end - ram_offset) assert struct.unpack('I', arm9.read(4))[0] == ARM9_BLZ_BEACON except AssertionError: # already decompressed arm9.seek(0) arm9dec.write(arm9.read()) return except struct.error: pass # at EOF reader = BinaryIO.reader(arm9) buff = decompress(reader, end - ram_offset) for i in range(game.load_info - ram_offset + 0x14, game.load_info - ram_offset + 0x18): buff[i] = 0 buff.tofile(arm9dec)
def decompress_overlays(game): """Creates an overarm9.dec.bin in the Game's workspace and an overlays_dez directory """ workspace = game.files.directory try: if os.path.getsize(os.path.join(workspace, 'overarm9.dec.bin')) > 0: return except OSError: pass try: os.mkdir(os.path.join(workspace, 'overlays_dez')) except: pass with open(os.path.join(workspace, 'header.bin')) as header: header.seek(0x54) size, = struct.unpack('I', header.read(4)) with open(os.path.join(workspace, 'overarm9.bin')) as overarm: ovt = OverlayTable(size >> 5, reader=overarm) for overlay in ovt.overlays: fname = os.path.join(workspace, 'overlays', 'overlay_{0:04}.bin'.format(overlay.file_id)) outname = os.path.join( workspace, 'overlays_dez', 'overlay_{0:04}.bin'.format(overlay.file_id)) if overlay.compressed: end = overlay.reserved & 0xFFFFFF with open(fname) as compressed_handle,\ open(outname, 'w') as decompressed_handle: reader = BinaryIO.reader(compressed_handle) buff = decompress(reader, end) buff.tofile(decompressed_handle) overlay.reserved = 0 else: shutil.copy2(fname, outname) with open(os.path.join(workspace, 'overarm9.dec.bin'), 'w') as overarm: ovt.save(overarm)
def extract_scripts(self, filename, output_folder, debug=False): self.offsets = [] self.scripts = [] self.functions = [] self.func_map = {} # {offset: [func, count]} output = "" print(filename) fin = open(os.path.join(self.config.path, filename), "rb+") fin = (BinaryIO.reader(fin.read())).adapter(fin) fin.seek(0) fin.seek(0, os.SEEK_END) filesize = fin.tell() offset = 0 num = 0 song_names = [''] export_sounds(fin, 0x0, song_names, filename, output_folder, 'Script') return (output, offset)
def load(self, reader): reader = BinaryIO.reader(reader) AtomicStruct.load(self, reader) self.files = {} self.ids = {} offsets = [] sizes = [] if self.version in game.GEN_IV: commented = False # (self.seed & 0x1FF) == 0x1FF for i in xrange(1, self.num + 1): state = (((self.seed * 0x2FD) & 0xFFFF) * i) & 0xFFFF key = state | state << 16 offsets.append(reader.readUInt32() ^ key) sizes.append(reader.readUInt32() ^ key) if commented: state = (((self.seed * 0x2FD) & 0xFFFF) * i) & 0xFFFF key = state | state << 16 comment_ofs = reader.readUInt32() ^ key term = reader.readUInt32() ^ key if term != 0xFFFF: raise ValueError('Expected 0xFFFF comment ofs terminator.' ' Got {0:#x}'.format(term)) for i in xrange(self.num): compressed = False reader.seek(offsets[i]) size = sizes[i] string = [] key = (TEXT_KEY4_INIT * (i + 1)) & 0xFFFF for j in range(size): string.append(reader.readUInt16() ^ key) key = (key + TEXT_KEY4_STEP) & 0xFFFF if string[0] == 0xF100: compressed = True string = decompress(string) text = '' while string: char = string.pop(0) if char == 0xFFFF: break elif char == 0xFFFE: text += 'VAR(' args = [string.pop(0)] count = string.pop(0) args += string[:count] string = string[count:] text += ', '.join(map(str, args)) text += ')' elif char == 0xE000: text += '\\n' elif char == 0x25bc: text += '\\r' elif char == 0x25bd: text += '\\f' else: try: text += table[char] except KeyError: text += '\\?{0:04x}'.format(char) name = '0_{0:05}'.format(i) if compressed: name += 'c' self.files[name] = text self.ids[i] = name else: raise RuntimeError('Did not have a terminating character') else: commented = False for i in xrange(self.numblocks): offsets.append(reader.readUInt32()) block = Editable() block.uint32('size') block.array('entries', TableEntry(self.version).base_struct, length=self.num) block.freeze() for i, block_offset in enumerate(offsets): reader.seek(block_offset) block.load(reader) for j, entry in enumerate(block.entries): compressed = False text = '' reader.seek(block_offset + entry.offset) encchars = [ reader.readUInt16() for k in xrange(entry.charcount) ] seed = key = encchars[-1] ^ 0xFFFF string = [] # decrypted chars while encchars: char = encchars.pop() ^ key key = ((key >> 3) | (key << 13)) & 0xFFFF string.insert(0, char) if string[0] == 0xF100: compressed = True string = decompress(string, 16) while string: char = string.pop(0) if char == 0xFFFF: break elif char == 0xFFFE: text += '\\n' elif char < 20 or char > 0xF000: text += '\\?{0:04X}'.format(char) elif char == 0xF000: kind = string.pop(0) count = string.pop(0) if kind == 0xbe00 and not count: text += '\\f' elif kind == 0xbe01 and not count: text += '\\r' else: text += 'VAR(' args = [kind] args += string[:count] string = string[count:] text += ', '.join(map(str, args)) text += ')' else: text += unichr(char) name = '{0}_{1:05}'.format(i, j) c = 65 for k in xrange(16): if (entry.flags >> k) & 0x1: name += ord(c + k) if compressed: name += 'c' name += '[{0:04X}]'.format(seed) self.files[name] = text self.ids[j] = name if commented: reader.seek(comment_ofs) num = reader.readUInt16() for i in xrange(num): commentid = reader.readUInt16() text = '' while True: char = reader.readUInt16() if char == 0xFFFF: break text += unichr(char) name = '0c_{0:05}'.format(commentid) self.files[name] = text return self
def load(self, reader): self.offsets = [] self.scripts = [] self.functions = [] self.func_map = {} # {offset: [func, count]} reader = BinaryIO.reader(reader) start = reader.tell() try: offset = reader.readUInt32() except: # Empty File. No script contents return while offset: abs_offset = offset+reader.tell() current_pos = reader.tell() for offset in self.offsets: if current_pos > offset: break self.offsets.append(abs_offset) try: offset = reader.readUInt32() except: # Exhaustive offset list: not a script return if offset & 0xFFFF == 0xFD13: break if not self.offsets: return for scrnum, offset in enumerate(self.offsets, self.script_start): with reader.seek(offset): script = ScriptDecompiler(reader, self) script.parse() script.header_lines.append('def script_{num}(engine):' .format(num=scrnum)) self.scripts.append(script) changed = True while changed: changed = False for offset, (func, count, func_id) in self.func_map.items(): if func is None: changed = True with reader.seek(offset): script = ScriptDecompiler(reader, self) script.parse() self.func_map[offset][0] = script cur_id = self.function_start embedded_functions = [] for (offset, (func, count, func_id)) in self.func_map.items(): if count > 1: func.header_lines.append('def func_{num}(engine):' .format(num=cur_id)) self.functions.append(func) self.func_map[offset][2] = cur_id cur_id += 1 else: embedded_functions.append(func) for script in itertools.chain(self.scripts, self.functions, embedded_functions): for expr in script: try: if expr.target.args[0].name != 'jump': continue except: continue offset = expr.target.args[0].args[0] func, count, func_id = self.func_map[offset] if func_id is None: func.indent = 0 else: end = func.lines[-1] func = script.func( 'call', 'func_{0}'.format(func_id), namespace='engine.') if end.is_return() and end.args and end.args != (None, ): func = script.end(func) expr.set_target(func)
def load(self, reader): reader = BinaryIO.reader(reader) AtomicStruct.load(self, reader) data = reader.read(self.size_ - 8) self.files.extend([data[entry] for entry in self.narc.fatb.entries_])
def load(self, reader): reader = BinaryIO.reader(reader) AtomicStruct.load(self, reader) for i in xrange(self._data.num): self.entries_.append( slice(reader.readUInt32(), reader.readUInt32()))
def learn(self, reader, methods): reader = BinaryIO.reader(reader) start = reader.tell() self._offsets = [] self._regions = {} def print_regions(): print('MAP START') regions_map = '' with reader.seek(start): until = 0 for i in xrange(max(self._regions) - 1): if i in self._regions: if self._regions[i] == -1: regions_map += '\033[91m{0:02X} \033[0m'.format( reader.readUInt8()) else: regions_map += '\033[92m{0:02X} \033[0m'.format( reader.readUInt8()) for j in xrange(1, self._regions[i]): regions_map += '\033[94m{0:02X} \033[0m'.format( reader.readUInt8()) until = i + j elif i > until: regions_map += '{0:02X} '.format(reader.readUInt8()) print(regions_map) def mark(size, advance=False): if advance: self._regions[reader.tell()] = size else: self._regions[reader.tell() - size] = size def check(size=None): reg = reader.tell() if size is None: return self._regions.get(reg, 0) is 0 elif not size: return True elif reg in self._regions: return self._regions[reg] == size for i in xrange(1, 16): try: if self._regions[reg - i] > i: return False break except: pass for i in xrange(1, size): if reg + i in self._regions: return False return True try: offset = reader.readUInt32() except: return while offset: # print(reader.tell(), self._regions) if not check(): break abs_offset = offset + reader.tell() self._offsets.append(abs_offset) mark(4) with reader.seek(abs_offset): mark(2, True) try: offset = reader.readUInt32() except: return if offset & 0xFFFF == 0xFD13: with reader.seek(reader.tell() - 2): mark(2) break if not self._offsets: return with reader.seek(reader.tell()): while True: try: reader.readUInt8() except: break if self._offsets[0] > reader.tell(): return mark(255, True) def parse(): if not check(2): return cmd = reader.readUInt16() mark(2) if not cmd: return if cmd in (0x2, 0x1b): return elif cmd in (0x16, 0x1A): offset = reader.readInt32() mark(4) with reader.seek(offset + reader.tell()): if check(): # Only need to check dest once parse() if cmd == 0x1A: parse() return elif cmd in (0x1C, 0x1D): arg = reader.readUInt8() mark(1) offset = reader.readInt32() mark(4) with reader.seek(offset + reader.tell()): if check(): # Only need to check dest once parse() parse() return elif cmd in (0x5E, ): arg = reader.readUInt16() mark(2) offset = reader.readInt32() mark(4) with reader.seek(offset + reader.tell()): arg = 0 while arg != 0xFE: if not check(2): raise Exception('Movement') arg = reader.readUInt16() mark(2) parse() return elif cmd == 0x1cf: arg = reader.readUInt8() mark(1) if arg == 2: reader.read(2) mark(2) parse() return elif cmd == 0x21d: arg = reader.readUInt16() mark(2) if arg in (4, 5): argsize = 4 elif arg == 6: argsize = 2 else: argsize = 6 argsize -= 2 # Already read two reader.read(argsize) if argsize: mark(argsize) parse() return elif cmd == 0x235: arg = reader.readUInt16() mark(2) if arg in (0, 6): argsize = 4 elif arg == 2: argsize = 2 elif arg == 4: argsize = 6 else: argsize = 8 argsize -= 2 # Already read two reader.read(argsize) if argsize: mark(argsize) parse() return elif cmd == 0x23e: arg = reader.readUInt16() mark(2) if arg in (1, 2, 3, 4): argsize = 4 elif arg in (5, 6): argsize = 6 else: argsize = 2 argsize -= 2 # Already read two reader.read(argsize) if argsize: mark(argsize) parse() return if cmd not in methods: method = Method(cmd) methods[cmd] = method else: method = methods[cmd] if method.known: argsize = method.argsize() if not check(argsize): mark(-1) print(cmd, argsize, reader.tell(), self._regions) print_regions() print(reader.tell()) raise Exception return reader.read(argsize) if argsize: mark(argsize) parse() return else: if cmd > 750: mark(-1) print(self._regions) print_regions() print(cmd, reader.tell()) raise Exception minbytes = 0 while True: if check(1): if not check(2): minbytes += 1 method.known = True if minbytes == 1: method.args = [1] else: method.maxbytes = method.minbytes = minbytes reader.readUInt8() mark(1) parse() return else: arg = reader.readUInt16() if arg & 0xC000 and not arg & 0x0F00: minbytes += 2 method.minbytes = max(minbytes, method.minbytes) else: break else: method.known = True if minbytes == 0: method.args = [] else: method.maxbytes = method.minbytes = minbytes parse() return okay = True for offset in self._offsets: with reader.seek(offset): # mark(2, True) try: parse() except: from util.code import print_helpful_traceback print_helpful_traceback() raise spaces = [] space_start = None reader.seek(start) for i in xrange(start, max(self._regions)): if check(1): if space_start is None: space_start = i space_end = i else: if space_start is not None: spaces.append([space_start, space_end + 1]) space_start = None reader.readUInt8() def check_parse(size, affinity=1.0): if not size: return 1 * affinity if not check(): return 1 * affinity if size == 1: if check(1): if reader.readUInt8(): return -1 * affinity size -= 2 if size < 0: return 0 if not check(2): # If we started inside of a parameter of the last, seek forward with reader.seek(reader.tell() + 1): return check_parse(size - 1, affinity) * affinity cmd = reader.readUInt16() if not cmd: return 0 if not size and cmd == 0x2: return 8 * affinity if cmd > 0x500: return -2 * affinity try: method = methods[cmd] except KeyError: method = Method(cmd) methods[cmd] = method if method.known: argsize = method.argsize() if size < argsize: return -2 * affinity reader.read(argsize) passed = 0 with reader.seek(reader.tell()): passed += check_parse(size - argsize, affinity) return passed * affinity elif method.minbytes: if size < method.minbytes: return -2 * affinity elif not size: method.add_form(weight=0.5) return 0.5 * affinity passed = 0 for k in xrange(method.minbytes, min(size + 1, MAX_ARGS)): with reader.seek(reader.tell()): reader.read(size - k) ret = check_parse(k, affinity / 2.0) method.add_form(*[1] * (size - k), weight=ret) passed += ret * .5 return passed / k * affinity print_regions() prev_end = 0 for space in spaces: if space[1] - space[0] < 16: # print('regions', self._regions) # print('offsets', self._offsets) for i in xrange(space[0], space[1]): for j in xrange(i, space[1]): # print(j, i, space) with reader.seek(j - 2): if reader.tell() == prev_end: maxbytes = space[1] - j with reader.seek(reader.tell()): cmd = reader.readUInt16() try: method = methods[cmd] except KeyError: method = Method(cmd) methods[cmd] = method if method.maxbytes > maxbytes: method.maxbytes = maxbytes method.prune() # with reader.seek(reader.tell()): # print([reader.readUInt8() for k in xrange(space[1]-j+2)]) check_parse(space[1] - j + 2) prev_end = space[1]
def load(self, reader): reader = BinaryIO.reader(reader) AtomicStruct.load(self, reader) self.fatb.load(reader) self.fntb.load(reader) self.fimg.load(reader)