def extract_scripts(self, filename, output_folder, debug=False):
self.offsets = []
self.scripts = []
self.functions = []
self.func_map = {} # {offset: [func, count]}
output = ""
print(filename)
fin = open(os.path.join(self.config.path, filename), "rb+")
fin = (BinaryIO.reader(fin.read())).adapter(fin)
fin.seek(0)
fin.seek(0, os.SEEK_END)
filesize = fin.tell()
offset = 0
num = 0
song_names = ['']
export_sounds(fin, 0x0, song_names, filename, output_folder, 'Script')
return (output, offset)
def load(self, reader):
reader = BinaryIO.reader(reader)
start = reader.tell()
self.magic = reader.read(4)
size = reader.readUInt32()
self.texinfo.load(reader)
self.tex4x4info.load(reader)
self.palinfo.load(reader)
# Build dicts
reader.seek(start + self.texinfo._lookupofs)
self.texdict.load(reader)
self.texparams = []
for i in xrange(self.texdict.num):
imgParam, extra = struct.unpack('II', self.texdict.data[i])
self.texparams.append(
TexParam((imgParam & 0xFFFF) << 3, 8 <<
((imgParam >> 20) & 0x7), 8 <<
((imgParam >> 23) & 0x7), (imgParam >> 26) & 0x7,
(imgParam >> 29) & 0x1))
reader.seek(start + self.palinfo._lookupofs)
self.paldict.load(reader)
self.palparams = []
for i in xrange(self.paldict.num):
offset, flag = struct.unpack('HH', self.paldict.data[i])
self.palparams.append(PalParam(offset << 3, flag))
# Read data.
reader.seek(start + self.texinfo._dataofs)
self.texdata = reader.read(self.texinfo._datasize)
reader.seek(start + self.palinfo._dataofs)
self.paldata = reader.read(self.palinfo._datasize)
# TODO 4x4
if size:
reader.seek(start + size)
self._images = None
def init(self, filename, base_address):
output = ""
self.filename = filename
self.file = open(filename, "rb+")
self.file = (BinaryIO.reader(self.file.read())).adapter(self.file)
self.file.seek(0)
self.base_address = base_address
def __init__(self, reader, compression=COMPRESSION_LZ77):
handle = BinaryIO.reader(reader)
start = handle.tell()
data = array.array('B', handle.read())
self.header = LZHeader._make([compression, len(data)])
handle.truncate(start)
handle.seek(start)
handle.write(pack('I', self.header.flag | (self.header.size << 8)))
out = array.array('B')
out.append(0)
out.extend(data[:4])
pos = 4
endpos = len(data)
search_start = 0
max_len = 0x12
control_bit = 4
flag = 0
flag_pos = 0
while pos < endpos:
best_pos = None
best_len = 0
for check in xrange(search_start, pos):
for idx in xrange(0, min(max_len + 1, pos - check,
endpos - pos)):
if data[check + idx] != data[pos + idx]:
break
if idx >= best_len:
best_len = idx
best_pos = check
if idx >= max_len:
break
control_bit -= 1
if best_len < 3:
out.append(data[pos])
pos += 1
else:
flag |= 1 << control_bit
head = (best_len - 3) << 0xC
head |= pos - best_pos - 1
out.append(head >> 8)
out.append(head & 0xFF)
pos += best_len
if control_bit <= 0:
if pos >= 0x400:
search_start = pos - 0x400
control_bit = 8
out[flag_pos] = flag
flag_pos = len(out)
flag = 0
out.append(0)
out[flag_pos] = flag
handle.write(out.tostring())
self.handle = handle
def add_binary(self, handle, name='.text', address=0x02000000):
section = Section(name, SectionHeader.TYPE_PROGBITS)
reader = BinaryIO.reader(handle)
with reader.seek(0):
section.data.write(reader.read())
section.header.address = address
section.header.flags = section.header.FLAG_EXECINSTR \
| section.header.FLAG_ALLOC
self.add_section(section)
symbol = self.add_symbol(name, address, Symbol.TYPE_SECTION)
symbol.type_ = symbol.TYPE_SECTION
def extract_scripts(filename, output_folder):
output = ""
print(filename)
fin = open(os.path.join(configuration.Config().path, filename), "rb+")
fin = (BinaryIO.reader(fin.read())).adapter(fin)
output = dumb_scripts(fin, 'Script')
head, tail = os.path.split(filename)
with open(os.path.join(output_folder + tail.replace(".bin", ".s")),
'w') as out:
out.write(output)
return output
def load(self, reader):
reader = BinaryIO.reader(reader)
start = reader.tell()
self.magic = reader.read(4)
self.endian = reader.readUInt16()
self.version = reader.readUInt16()
size = reader.readUInt32()
headersize = reader.readUInt16()
numblocks = reader.readUInt16()
if headersize:
reader.seek(start + headersize)
self.fatb.load(reader)
self.fntb.load(reader)
self.fimg.load(reader)
if size:
reader.seek(start + size)
def __init__(self, reader):
handle = BinaryIO.reader(reader)
start = handle.tell()
raw_header = unpack('I', handle.read(4))[0]
self.header = LZHeader._make([raw_header & 0xFF, raw_header >> 8])
if self.header.flag == 0x11:
lz_ss = True
elif self.header.flag == 0x10:
lz_ss = False
else:
raise ValueError('Invalid compression flag: {0}'.format(
self.header.flag))
out = []
while len(out) < self.header.size:
flag = unpack('B', handle.read(1))[0]
for x in range(7, -1, -1):
if len(out) >= self.header.size:
break
if not (flag >> x) & 0x1:
r = handle.read(1)
out.append(r)
continue
head, = unpack('>H', handle.read(2))
if not lz_ss:
count = ((head >> 12) & 0xF) + 3
back = head & 0xFFF
else:
ind = (head >> 12) & 0xF
if not ind:
tail = unpack('B', handle.read(1))[0]
count = (head >> 4) + 0x11
back = ((head & 0xF) << 8) | tail
elif ind == 1:
tail = unpack('>H', handle.read(2))[0]
count = (((head & 0xFFF) << 4) | (tail >> 12)) + 0x111
back = tail & 0xFFF
else:
count = ind + 1
back = head & 0xFFF
cursz = len(out)
for y in range(count):
out.append(out[cursz - back - 1 + y])
self.data = "".join(out)
handle.seek(start)
handle.write(self.data)
handle.seek(start)
self.handle = handle
def load(self, reader):
reader = BinaryIO.reader(reader)
start = reader.tell()
self.magic = reader.read(4)
headersize = reader.readUInt32()
u1 = reader.readUInt32()
u2 = reader.readUInt32()
u3 = reader.readUInt32()
size = reader.readUInt32()
u4 = reader.readUInt32()
if headersize:
reader.seek(start + headersize)
self.fato.load(reader)
self.fatb.load(reader)
self.fimb.load(reader)
if size:
reader.seek(start + size)
def load(self, reader):
reader = BinaryIO.reader(reader)
self.conditions = []
self.__getitem__ = self.conditions.__getitem__
self.__setitem__ = self.conditions.__setitem__
self.u0 = reader.readUInt16()
self.u2 = reader.readUInt16()
try:
self.u4 = reader.readUInt16()
except:
self.u4 = 0
return
while True:
try:
cond = ScriptCondition()
cond.load(reader)
self.conditions.append(cond)
except:
break
def decompress_arm9(game):
"""Creates an arm9.dec.bin in the Game's workspace
This file will be created even if arm9.bin is already decompressed
"""
workspace = game.files.directory
try:
if os.path.getsize(os.path.join(workspace, 'arm9.dec.bin')) > 0:
return
except OSError:
pass
with open(os.path.join(workspace, 'header.bin')) as header:
header.seek(0x24)
entry, ram_offset, size = struct.unpack('III', header.read(12))
with open(os.path.join(workspace, 'arm9.bin')) as arm9,\
open(os.path.join(workspace, 'arm9.dec.bin'), 'w') as arm9dec:
arm9.seek(game.load_info - ram_offset + 0x14)
end, u18, beacon, unbeacon = struct.unpack('IIII', arm9.read(16))
assert beacon & 0xFFFF0000 == ARM9_BLZ_BEACON & 0xFFFF0000
assert unbeacon & 0xFFFF == ARM9_BLZ_UNBEACON & 0xFFFF
# TODO: if beacons do not match, scan ARM9 for beacon
try:
assert end
arm9.seek(end - ram_offset)
assert struct.unpack('I', arm9.read(4))[0] == ARM9_BLZ_BEACON
except AssertionError:
# already decompressed
arm9.seek(0)
arm9dec.write(arm9.read())
return
except struct.error:
pass # at EOF
reader = BinaryIO.reader(arm9)
buff = decompress(reader, end - ram_offset)
for i in range(game.load_info - ram_offset + 0x14,
game.load_info - ram_offset + 0x18):
buff[i] = 0
buff.tofile(arm9dec)
def decompress_overlays(game):
"""Creates an overarm9.dec.bin in the Game's workspace and
an overlays_dez directory
"""
workspace = game.files.directory
try:
if os.path.getsize(os.path.join(workspace, 'overarm9.dec.bin')) > 0:
return
except OSError:
pass
try:
os.mkdir(os.path.join(workspace, 'overlays_dez'))
except:
pass
with open(os.path.join(workspace, 'header.bin')) as header:
header.seek(0x54)
size, = struct.unpack('I', header.read(4))
with open(os.path.join(workspace, 'overarm9.bin')) as overarm:
ovt = OverlayTable(size >> 5, reader=overarm)
for overlay in ovt.overlays:
fname = os.path.join(workspace, 'overlays',
'overlay_{0:04}.bin'.format(overlay.file_id))
outname = os.path.join(
workspace, 'overlays_dez',
'overlay_{0:04}.bin'.format(overlay.file_id))
if overlay.compressed:
end = overlay.reserved & 0xFFFFFF
with open(fname) as compressed_handle,\
open(outname, 'w') as decompressed_handle:
reader = BinaryIO.reader(compressed_handle)
buff = decompress(reader, end)
buff.tofile(decompressed_handle)
overlay.reserved = 0
else:
shutil.copy2(fname, outname)
with open(os.path.join(workspace, 'overarm9.dec.bin'), 'w') as overarm:
ovt.save(overarm)
def extract_scripts(self, filename, output_folder, debug=False):
self.offsets = []
self.scripts = []
self.functions = []
self.func_map = {} # {offset: [func, count]}
output = ""
print(filename)
fin = open(os.path.join(self.config.path, filename), "rb+")
fin = (BinaryIO.reader(fin.read())).adapter(fin)
fin.seek(0)
fin.seek(0, os.SEEK_END)
filesize = fin.tell()
offset = 0
num = 0
song_names = ['']
export_sounds(fin, 0x0, song_names, filename, output_folder, 'Script')
return (output, offset)
def load(self, reader):
reader = BinaryIO.reader(reader)
AtomicStruct.load(self, reader)
self.files = {}
self.ids = {}
offsets = []
sizes = []
if self.version in game.GEN_IV:
commented = False # (self.seed & 0x1FF) == 0x1FF
for i in xrange(1, self.num + 1):
state = (((self.seed * 0x2FD) & 0xFFFF) * i) & 0xFFFF
key = state | state << 16
offsets.append(reader.readUInt32() ^ key)
sizes.append(reader.readUInt32() ^ key)
if commented:
state = (((self.seed * 0x2FD) & 0xFFFF) * i) & 0xFFFF
key = state | state << 16
comment_ofs = reader.readUInt32() ^ key
term = reader.readUInt32() ^ key
if term != 0xFFFF:
raise ValueError('Expected 0xFFFF comment ofs terminator.'
' Got {0:#x}'.format(term))
for i in xrange(self.num):
compressed = False
reader.seek(offsets[i])
size = sizes[i]
string = []
key = (TEXT_KEY4_INIT * (i + 1)) & 0xFFFF
for j in range(size):
string.append(reader.readUInt16() ^ key)
key = (key + TEXT_KEY4_STEP) & 0xFFFF
if string[0] == 0xF100:
compressed = True
string = decompress(string)
text = ''
while string:
char = string.pop(0)
if char == 0xFFFF:
break
elif char == 0xFFFE:
text += 'VAR('
args = [string.pop(0)]
count = string.pop(0)
args += string[:count]
string = string[count:]
text += ', '.join(map(str, args))
text += ')'
elif char == 0xE000:
text += '\\n'
elif char == 0x25bc:
text += '\\r'
elif char == 0x25bd:
text += '\\f'
else:
try:
text += table[char]
except KeyError:
text += '\\?{0:04x}'.format(char)
name = '0_{0:05}'.format(i)
if compressed:
name += 'c'
self.files[name] = text
self.ids[i] = name
else:
raise RuntimeError('Did not have a terminating character')
else:
commented = False
for i in xrange(self.numblocks):
offsets.append(reader.readUInt32())
block = Editable()
block.uint32('size')
block.array('entries',
TableEntry(self.version).base_struct,
length=self.num)
block.freeze()
for i, block_offset in enumerate(offsets):
reader.seek(block_offset)
block.load(reader)
for j, entry in enumerate(block.entries):
compressed = False
text = ''
reader.seek(block_offset + entry.offset)
encchars = [
reader.readUInt16() for k in xrange(entry.charcount)
]
seed = key = encchars[-1] ^ 0xFFFF
string = [] # decrypted chars
while encchars:
char = encchars.pop() ^ key
key = ((key >> 3) | (key << 13)) & 0xFFFF
string.insert(0, char)
if string[0] == 0xF100:
compressed = True
string = decompress(string, 16)
while string:
char = string.pop(0)
if char == 0xFFFF:
break
elif char == 0xFFFE:
text += '\\n'
elif char < 20 or char > 0xF000:
text += '\\?{0:04X}'.format(char)
elif char == 0xF000:
kind = string.pop(0)
count = string.pop(0)
if kind == 0xbe00 and not count:
text += '\\f'
elif kind == 0xbe01 and not count:
text += '\\r'
else:
text += 'VAR('
args = [kind]
args += string[:count]
string = string[count:]
text += ', '.join(map(str, args))
text += ')'
else:
text += unichr(char)
name = '{0}_{1:05}'.format(i, j)
c = 65
for k in xrange(16):
if (entry.flags >> k) & 0x1:
name += ord(c + k)
if compressed:
name += 'c'
name += '[{0:04X}]'.format(seed)
self.files[name] = text
self.ids[j] = name
if commented:
reader.seek(comment_ofs)
num = reader.readUInt16()
for i in xrange(num):
commentid = reader.readUInt16()
text = ''
while True:
char = reader.readUInt16()
if char == 0xFFFF:
break
text += unichr(char)
name = '0c_{0:05}'.format(commentid)
self.files[name] = text
return self
def load(self, reader):
self.offsets = []
self.scripts = []
self.functions = []
self.func_map = {} # {offset: [func, count]}
reader = BinaryIO.reader(reader)
start = reader.tell()
try:
offset = reader.readUInt32()
except:
# Empty File. No script contents
return
while offset:
abs_offset = offset+reader.tell()
current_pos = reader.tell()
for offset in self.offsets:
if current_pos > offset:
break
self.offsets.append(abs_offset)
try:
offset = reader.readUInt32()
except:
# Exhaustive offset list: not a script
return
if offset & 0xFFFF == 0xFD13:
break
if not self.offsets:
return
for scrnum, offset in enumerate(self.offsets, self.script_start):
with reader.seek(offset):
script = ScriptDecompiler(reader, self)
script.parse()
script.header_lines.append('def script_{num}(engine):'
.format(num=scrnum))
self.scripts.append(script)
changed = True
while changed:
changed = False
for offset, (func, count, func_id) in self.func_map.items():
if func is None:
changed = True
with reader.seek(offset):
script = ScriptDecompiler(reader, self)
script.parse()
self.func_map[offset][0] = script
cur_id = self.function_start
embedded_functions = []
for (offset, (func, count, func_id)) in self.func_map.items():
if count > 1:
func.header_lines.append('def func_{num}(engine):'
.format(num=cur_id))
self.functions.append(func)
self.func_map[offset][2] = cur_id
cur_id += 1
else:
embedded_functions.append(func)
for script in itertools.chain(self.scripts, self.functions,
embedded_functions):
for expr in script:
try:
if expr.target.args[0].name != 'jump':
continue
except:
continue
offset = expr.target.args[0].args[0]
func, count, func_id = self.func_map[offset]
if func_id is None:
func.indent = 0
else:
end = func.lines[-1]
func = script.func(
'call', 'func_{0}'.format(func_id),
namespace='engine.')
if end.is_return() and end.args and end.args != (None, ):
func = script.end(func)
expr.set_target(func)
def load(self, reader):
reader = BinaryIO.reader(reader)
AtomicStruct.load(self, reader)
data = reader.read(self.size_ - 8)
self.files.extend([data[entry] for entry in self.narc.fatb.entries_])
def load(self, reader):
reader = BinaryIO.reader(reader)
AtomicStruct.load(self, reader)
for i in xrange(self._data.num):
self.entries_.append(
slice(reader.readUInt32(), reader.readUInt32()))
def learn(self, reader, methods):
reader = BinaryIO.reader(reader)
start = reader.tell()
self._offsets = []
self._regions = {}
def print_regions():
print('MAP START')
regions_map = ''
with reader.seek(start):
until = 0
for i in xrange(max(self._regions) - 1):
if i in self._regions:
if self._regions[i] == -1:
regions_map += '\033[91m{0:02X} \033[0m'.format(
reader.readUInt8())
else:
regions_map += '\033[92m{0:02X} \033[0m'.format(
reader.readUInt8())
for j in xrange(1, self._regions[i]):
regions_map += '\033[94m{0:02X} \033[0m'.format(
reader.readUInt8())
until = i + j
elif i > until:
regions_map += '{0:02X} '.format(reader.readUInt8())
print(regions_map)
def mark(size, advance=False):
if advance:
self._regions[reader.tell()] = size
else:
self._regions[reader.tell() - size] = size
def check(size=None):
reg = reader.tell()
if size is None:
return self._regions.get(reg, 0) is 0
elif not size:
return True
elif reg in self._regions:
return self._regions[reg] == size
for i in xrange(1, 16):
try:
if self._regions[reg - i] > i:
return False
break
except:
pass
for i in xrange(1, size):
if reg + i in self._regions:
return False
return True
try:
offset = reader.readUInt32()
except:
return
while offset:
# print(reader.tell(), self._regions)
if not check():
break
abs_offset = offset + reader.tell()
self._offsets.append(abs_offset)
mark(4)
with reader.seek(abs_offset):
mark(2, True)
try:
offset = reader.readUInt32()
except:
return
if offset & 0xFFFF == 0xFD13:
with reader.seek(reader.tell() - 2):
mark(2)
break
if not self._offsets:
return
with reader.seek(reader.tell()):
while True:
try:
reader.readUInt8()
except:
break
if self._offsets[0] > reader.tell():
return
mark(255, True)
def parse():
if not check(2):
return
cmd = reader.readUInt16()
mark(2)
if not cmd:
return
if cmd in (0x2, 0x1b):
return
elif cmd in (0x16, 0x1A):
offset = reader.readInt32()
mark(4)
with reader.seek(offset + reader.tell()):
if check(): # Only need to check dest once
parse()
if cmd == 0x1A:
parse()
return
elif cmd in (0x1C, 0x1D):
arg = reader.readUInt8()
mark(1)
offset = reader.readInt32()
mark(4)
with reader.seek(offset + reader.tell()):
if check(): # Only need to check dest once
parse()
parse()
return
elif cmd in (0x5E, ):
arg = reader.readUInt16()
mark(2)
offset = reader.readInt32()
mark(4)
with reader.seek(offset + reader.tell()):
arg = 0
while arg != 0xFE:
if not check(2):
raise Exception('Movement')
arg = reader.readUInt16()
mark(2)
parse()
return
elif cmd == 0x1cf:
arg = reader.readUInt8()
mark(1)
if arg == 2:
reader.read(2)
mark(2)
parse()
return
elif cmd == 0x21d:
arg = reader.readUInt16()
mark(2)
if arg in (4, 5):
argsize = 4
elif arg == 6:
argsize = 2
else:
argsize = 6
argsize -= 2 # Already read two
reader.read(argsize)
if argsize:
mark(argsize)
parse()
return
elif cmd == 0x235:
arg = reader.readUInt16()
mark(2)
if arg in (0, 6):
argsize = 4
elif arg == 2:
argsize = 2
elif arg == 4:
argsize = 6
else:
argsize = 8
argsize -= 2 # Already read two
reader.read(argsize)
if argsize:
mark(argsize)
parse()
return
elif cmd == 0x23e:
arg = reader.readUInt16()
mark(2)
if arg in (1, 2, 3, 4):
argsize = 4
elif arg in (5, 6):
argsize = 6
else:
argsize = 2
argsize -= 2 # Already read two
reader.read(argsize)
if argsize:
mark(argsize)
parse()
return
if cmd not in methods:
method = Method(cmd)
methods[cmd] = method
else:
method = methods[cmd]
if method.known:
argsize = method.argsize()
if not check(argsize):
mark(-1)
print(cmd, argsize, reader.tell(), self._regions)
print_regions()
print(reader.tell())
raise Exception
return
reader.read(argsize)
if argsize:
mark(argsize)
parse()
return
else:
if cmd > 750:
mark(-1)
print(self._regions)
print_regions()
print(cmd, reader.tell())
raise Exception
minbytes = 0
while True:
if check(1):
if not check(2):
minbytes += 1
method.known = True
if minbytes == 1:
method.args = [1]
else:
method.maxbytes = method.minbytes = minbytes
reader.readUInt8()
mark(1)
parse()
return
else:
arg = reader.readUInt16()
if arg & 0xC000 and not arg & 0x0F00:
minbytes += 2
method.minbytes = max(minbytes, method.minbytes)
else:
break
else:
method.known = True
if minbytes == 0:
method.args = []
else:
method.maxbytes = method.minbytes = minbytes
parse()
return
okay = True
for offset in self._offsets:
with reader.seek(offset):
# mark(2, True)
try:
parse()
except:
from util.code import print_helpful_traceback
print_helpful_traceback()
raise
spaces = []
space_start = None
reader.seek(start)
for i in xrange(start, max(self._regions)):
if check(1):
if space_start is None:
space_start = i
space_end = i
else:
if space_start is not None:
spaces.append([space_start, space_end + 1])
space_start = None
reader.readUInt8()
def check_parse(size, affinity=1.0):
if not size:
return 1 * affinity
if not check():
return 1 * affinity
if size == 1:
if check(1):
if reader.readUInt8():
return -1 * affinity
size -= 2
if size < 0:
return 0
if not check(2):
# If we started inside of a parameter of the last, seek forward
with reader.seek(reader.tell() + 1):
return check_parse(size - 1, affinity) * affinity
cmd = reader.readUInt16()
if not cmd:
return 0
if not size and cmd == 0x2:
return 8 * affinity
if cmd > 0x500:
return -2 * affinity
try:
method = methods[cmd]
except KeyError:
method = Method(cmd)
methods[cmd] = method
if method.known:
argsize = method.argsize()
if size < argsize:
return -2 * affinity
reader.read(argsize)
passed = 0
with reader.seek(reader.tell()):
passed += check_parse(size - argsize, affinity)
return passed * affinity
elif method.minbytes:
if size < method.minbytes:
return -2 * affinity
elif not size:
method.add_form(weight=0.5)
return 0.5 * affinity
passed = 0
for k in xrange(method.minbytes, min(size + 1, MAX_ARGS)):
with reader.seek(reader.tell()):
reader.read(size - k)
ret = check_parse(k, affinity / 2.0)
method.add_form(*[1] * (size - k), weight=ret)
passed += ret * .5
return passed / k * affinity
print_regions()
prev_end = 0
for space in spaces:
if space[1] - space[0] < 16:
# print('regions', self._regions)
# print('offsets', self._offsets)
for i in xrange(space[0], space[1]):
for j in xrange(i, space[1]):
# print(j, i, space)
with reader.seek(j - 2):
if reader.tell() == prev_end:
maxbytes = space[1] - j
with reader.seek(reader.tell()):
cmd = reader.readUInt16()
try:
method = methods[cmd]
except KeyError:
method = Method(cmd)
methods[cmd] = method
if method.maxbytes > maxbytes:
method.maxbytes = maxbytes
method.prune()
# with reader.seek(reader.tell()):
# print([reader.readUInt8() for k in xrange(space[1]-j+2)])
check_parse(space[1] - j + 2)
prev_end = space[1]
def load(self, reader):
reader = BinaryIO.reader(reader)
AtomicStruct.load(self, reader)
self.fatb.load(reader)
self.fntb.load(reader)
self.fimg.load(reader)
