def __init__(self, name, type=0):
self.header = SectionHeader(name)
self.data = BinaryIO()
self.header.type_ = type
if type == SectionHeader.TYPE_STRTAB:
self.data.writeUInt8(0)
elif type == SectionHeader.TYPE_SYMTAB:
self.header.entsize = Symbol(None).get_size()
def load(self, reader):
reader = BinaryIO.reader(reader)
start = reader.tell()
self.magic = reader.read(4)
size = reader.readUInt32()
self.texinfo.load(reader)
self.tex4x4info.load(reader)
self.palinfo.load(reader)
# Build dicts
reader.seek(start + self.texinfo._lookupofs)
self.texdict.load(reader)
self.texparams = []
for i in xrange(self.texdict.num):
imgParam, extra = struct.unpack('II', self.texdict.data[i])
self.texparams.append(
TexParam((imgParam & 0xFFFF) << 3, 8 <<
((imgParam >> 20) & 0x7), 8 <<
((imgParam >> 23) & 0x7), (imgParam >> 26) & 0x7,
(imgParam >> 29) & 0x1))
reader.seek(start + self.palinfo._lookupofs)
self.paldict.load(reader)
self.palparams = []
for i in xrange(self.paldict.num):
offset, flag = struct.unpack('HH', self.paldict.data[i])
self.palparams.append(PalParam(offset << 3, flag))
# Read data.
reader.seek(start + self.texinfo._dataofs)
self.texdata = reader.read(self.texinfo._datasize)
reader.seek(start + self.palinfo._dataofs)
self.paldata = reader.read(self.palinfo._datasize)
# TODO 4x4
if size:
reader.seek(start + size)
self._images = None
def extract_scripts(self, filename, output_folder, debug=False):
self.offsets = []
self.scripts = []
self.functions = []
self.func_map = {} # {offset: [func, count]}
output = ""
print(filename)
fin = open(os.path.join(self.config.path, filename), "rb+")
fin = (BinaryIO.reader(fin.read())).adapter(fin)
fin.seek(0)
fin.seek(0, os.SEEK_END)
filesize = fin.tell()
offset = 0
num = 0
song_names = ['']
export_sounds(fin, 0x0, song_names, filename, output_folder, 'Script')
return (output, offset)
def init(self, filename, base_address):
output = ""
self.filename = filename
self.file = open(filename, "rb+")
self.file = (BinaryIO.reader(self.file.read())).adapter(self.file)
self.file.seek(0)
self.base_address = base_address
def save(self, writer=None):
writer = BinaryIO.writer(writer)
writer.writeUInt16(self.u0)
writer.writeUInt16(self.u2)
if self.conditions:
writer.writeUInt16(self.u4)
for cond in self.conditions:
writer = cond.save(writer)
writer.writeAlign(4)
return writer
def save(self, writer=None):
writer = BinaryIO.writer(writer)
start = writer.tell()
writer = Editable.save(self, writer)
writer = self.char.save(writer)
if self.cpos.loaded:
writer = self.cpos.save(writer)
size = writer.tell() - start
with writer.seek(start + self.get_offset('size_')):
writer.writeUInt32(size)
return writer
def __init__(self, reader, compression=COMPRESSION_LZ77):
handle = BinaryIO.reader(reader)
start = handle.tell()
data = array.array('B', handle.read())
self.header = LZHeader._make([compression, len(data)])
handle.truncate(start)
handle.seek(start)
handle.write(pack('I', self.header.flag | (self.header.size << 8)))
out = array.array('B')
out.append(0)
out.extend(data[:4])
pos = 4
endpos = len(data)
search_start = 0
max_len = 0x12
control_bit = 4
flag = 0
flag_pos = 0
while pos < endpos:
best_pos = None
best_len = 0
for check in xrange(search_start, pos):
for idx in xrange(0, min(max_len + 1, pos - check,
endpos - pos)):
if data[check + idx] != data[pos + idx]:
break
if idx >= best_len:
best_len = idx
best_pos = check
if idx >= max_len:
break
control_bit -= 1
if best_len < 3:
out.append(data[pos])
pos += 1
else:
flag |= 1 << control_bit
head = (best_len - 3) << 0xC
head |= pos - best_pos - 1
out.append(head >> 8)
out.append(head & 0xFF)
pos += best_len
if control_bit <= 0:
if pos >= 0x400:
search_start = pos - 0x400
control_bit = 8
out[flag_pos] = flag
flag_pos = len(out)
flag = 0
out.append(0)
out[flag_pos] = flag
handle.write(out.tostring())
self.handle = handle
def add_binary(self, handle, name='.text', address=0x02000000):
section = Section(name, SectionHeader.TYPE_PROGBITS)
reader = BinaryIO.reader(handle)
with reader.seek(0):
section.data.write(reader.read())
section.header.address = address
section.header.flags = section.header.FLAG_EXECINSTR \
| section.header.FLAG_ALLOC
self.add_section(section)
symbol = self.add_symbol(name, address, Symbol.TYPE_SECTION)
symbol.type_ = symbol.TYPE_SECTION
def save(self, writer=None):
writer = BinaryIO.writer(writer)
start = writer.tell()
writer = Editable.save(self, writer)
writer = self.fatb.save(writer)
writer = self.fntb.save(writer)
writer = self.fimg.save(writer)
writer.writeAlign(4)
size = writer.tell() - start
with writer.seek(start + self.get_offset('size_')):
writer.writeUInt32(size)
return writer
def extract_scripts(filename, output_folder):
output = ""
print(filename)
fin = open(os.path.join(configuration.Config().path, filename), "rb+")
fin = (BinaryIO.reader(fin.read())).adapter(fin)
output = dumb_scripts(fin, 'Script')
head, tail = os.path.split(filename)
with open(os.path.join(output_folder + tail.replace(".bin", ".s")),
'w') as out:
out.write(output)
return output
def load(self, reader):
reader = BinaryIO.reader(reader)
start = reader.tell()
self.magic = reader.read(4)
self.endian = reader.readUInt16()
self.version = reader.readUInt16()
size = reader.readUInt32()
headersize = reader.readUInt16()
numblocks = reader.readUInt16()
if headersize:
reader.seek(start + headersize)
self.fatb.load(reader)
self.fntb.load(reader)
self.fimg.load(reader)
if size:
reader.seek(start + size)
def __init__(self, reader):
handle = BinaryIO.reader(reader)
start = handle.tell()
raw_header = unpack('I', handle.read(4))[0]
self.header = LZHeader._make([raw_header & 0xFF, raw_header >> 8])
if self.header.flag == 0x11:
lz_ss = True
elif self.header.flag == 0x10:
lz_ss = False
else:
raise ValueError('Invalid compression flag: {0}'.format(
self.header.flag))
out = []
while len(out) < self.header.size:
flag = unpack('B', handle.read(1))[0]
for x in range(7, -1, -1):
if len(out) >= self.header.size:
break
if not (flag >> x) & 0x1:
r = handle.read(1)
out.append(r)
continue
head, = unpack('>H', handle.read(2))
if not lz_ss:
count = ((head >> 12) & 0xF) + 3
back = head & 0xFFF
else:
ind = (head >> 12) & 0xF
if not ind:
tail = unpack('B', handle.read(1))[0]
count = (head >> 4) + 0x11
back = ((head & 0xF) << 8) | tail
elif ind == 1:
tail = unpack('>H', handle.read(2))[0]
count = (((head & 0xFFF) << 4) | (tail >> 12)) + 0x111
back = tail & 0xFFF
else:
count = ind + 1
back = head & 0xFFF
cursz = len(out)
for y in range(count):
out.append(out[cursz - back - 1 + y])
self.data = "".join(out)
handle.seek(start)
handle.write(self.data)
handle.seek(start)
self.handle = handle
def load(self, reader):
reader = BinaryIO.reader(reader)
start = reader.tell()
self.magic = reader.read(4)
headersize = reader.readUInt32()
u1 = reader.readUInt32()
u2 = reader.readUInt32()
u3 = reader.readUInt32()
size = reader.readUInt32()
u4 = reader.readUInt32()
if headersize:
reader.seek(start + headersize)
self.fato.load(reader)
self.fatb.load(reader)
self.fimb.load(reader)
if size:
reader.seek(start + size)
def load(self, reader):
reader = BinaryIO.reader(reader)
self.conditions = []
self.__getitem__ = self.conditions.__getitem__
self.__setitem__ = self.conditions.__setitem__
self.u0 = reader.readUInt16()
self.u2 = reader.readUInt16()
try:
self.u4 = reader.readUInt16()
except:
self.u4 = 0
return
while True:
try:
cond = ScriptCondition()
cond.load(reader)
self.conditions.append(cond)
except:
break
def to_file(self, handle):
"""Writes ELF to a writable file
Parameters
----------
handle : writable
Output to write to
"""
writer = BinaryIO.adapter(handle)
writer.writeUInt32(ELF_MAGIC)
writer.writeUInt8(1) # 32 bit
writer.writeUInt8(1) # little endian
writer.writeUInt8(1) # version
writer.writeUInt8(0) # ABI (None set)
writer.writeUInt8(0) # ABI (None set)
writer.writePadding(0x10, chr(0)) # Padding - 7 bytes
writer.writeUInt16(2) # Executable?
writer.writeUInt16(0x28) # ARM architecture
writer.writeUInt32(1) # version
writer.writeUInt32(self.entry)
writer.writeUInt32(0) # phoff?
shoff_ofs = writer.tell()
writer.writeUInt32(0) # section header offset
writer.writeUInt32(0) # flags
writer.writeUInt16(0x34) # header size
writer.writeUInt16(0) # phentsize?
writer.writeUInt16(0) # phnum?
writer.writeUInt16(self.sections[0].header.get_size()) # shentsize
writer.writeUInt16(len(self.sections)) # shnum
writer.writeUInt16(1) # shstrndx
for section in self.sections:
with section.data.seek(0):
section.header.offset = writer.tell()
writer.write(section.data.read())
section.header.size_ = section.data.tell()
writer.writeAlign(4)
shoff = writer.tell()
with writer.seek(shoff_ofs):
writer.writeUInt32(shoff)
for section in self.sections:
writer = section.header.save(writer)
def decompress_arm9(game):
"""Creates an arm9.dec.bin in the Game's workspace
This file will be created even if arm9.bin is already decompressed
"""
workspace = game.files.directory
try:
if os.path.getsize(os.path.join(workspace, 'arm9.dec.bin')) > 0:
return
except OSError:
pass
with open(os.path.join(workspace, 'header.bin')) as header:
header.seek(0x24)
entry, ram_offset, size = struct.unpack('III', header.read(12))
with open(os.path.join(workspace, 'arm9.bin')) as arm9,\
open(os.path.join(workspace, 'arm9.dec.bin'), 'w') as arm9dec:
arm9.seek(game.load_info - ram_offset + 0x14)
end, u18, beacon, unbeacon = struct.unpack('IIII', arm9.read(16))
assert beacon & 0xFFFF0000 == ARM9_BLZ_BEACON & 0xFFFF0000
assert unbeacon & 0xFFFF == ARM9_BLZ_UNBEACON & 0xFFFF
# TODO: if beacons do not match, scan ARM9 for beacon
try:
assert end
arm9.seek(end - ram_offset)
assert struct.unpack('I', arm9.read(4))[0] == ARM9_BLZ_BEACON
except AssertionError:
# already decompressed
arm9.seek(0)
arm9dec.write(arm9.read())
return
except struct.error:
pass # at EOF
reader = BinaryIO.reader(arm9)
buff = decompress(reader, end - ram_offset)
for i in range(game.load_info - ram_offset + 0x14,
game.load_info - ram_offset + 0x18):
buff[i] = 0
buff.tofile(arm9dec)
def decompress_overlays(game):
"""Creates an overarm9.dec.bin in the Game's workspace and
an overlays_dez directory
"""
workspace = game.files.directory
try:
if os.path.getsize(os.path.join(workspace, 'overarm9.dec.bin')) > 0:
return
except OSError:
pass
try:
os.mkdir(os.path.join(workspace, 'overlays_dez'))
except:
pass
with open(os.path.join(workspace, 'header.bin')) as header:
header.seek(0x54)
size, = struct.unpack('I', header.read(4))
with open(os.path.join(workspace, 'overarm9.bin')) as overarm:
ovt = OverlayTable(size >> 5, reader=overarm)
for overlay in ovt.overlays:
fname = os.path.join(workspace, 'overlays',
'overlay_{0:04}.bin'.format(overlay.file_id))
outname = os.path.join(
workspace, 'overlays_dez',
'overlay_{0:04}.bin'.format(overlay.file_id))
if overlay.compressed:
end = overlay.reserved & 0xFFFFFF
with open(fname) as compressed_handle,\
open(outname, 'w') as decompressed_handle:
reader = BinaryIO.reader(compressed_handle)
buff = decompress(reader, end)
buff.tofile(decompressed_handle)
overlay.reserved = 0
else:
shutil.copy2(fname, outname)
with open(os.path.join(workspace, 'overarm9.dec.bin'), 'w') as overarm:
ovt.save(overarm)
def save(self, writer=None):
"""
When writing the general structure, it saves offsets
to unknown values. It is the duty of the parent to
fill these in
"""
writer = writer if writer is not None else BinaryIO()
writer.writeUInt32(self.vramkey)
self._datasize_ofs = writer.tell()
writer.writeUInt16(0) # datasize
if self.infotype != TexInfo.INFO_PAL:
self._lookupofs_ofs = writer.tell()
writer.writeUInt16(0)
writer.writeUInt16(0)
if self.infotype == TexInfo.INFO_PAL:
self._lookupofs_ofs = writer.tell()
writer.writeUInt16(0)
writer.writeUInt16(8)
self._dataofs_ofs = writer.tell()
writer.writeUInt32(0)
if self.infotype == TexInfo.INFO_TEX4X4:
writer.writeUInt32(0)
return writer
def extract_scripts(self, filename, output_folder, debug=False):
self.offsets = []
self.scripts = []
self.functions = []
self.func_map = {} # {offset: [func, count]}
output = ""
print(filename)
fin = open(os.path.join(self.config.path, filename), "rb+")
fin = (BinaryIO.reader(fin.read())).adapter(fin)
fin.seek(0)
fin.seek(0, os.SEEK_END)
filesize = fin.tell()
offset = 0
num = 0
song_names = ['']
export_sounds(fin, 0x0, song_names, filename, output_folder, 'Script')
return (output, offset)
def save(self, writer=None):
if writer is None:
writer = BinaryIO()
start = writer.tell()
writer.write(self.magic)
sizeofs = writer.tell()
writer.writeUInt32(0)
writer.writeUInt32(4)
writer.writeUInt32(0x10000)
size = writer.tell() - start
with writer.seek(sizeofs):
writer.writeUInt32(size)
return writer
def save(self, writer=None):
if writer is None:
writer = BinaryIO()
start = writer.tell()
writer.write(self.magic)
sizeofs = writer.tell()
writer.writeUInt32(0)
writer.writeUInt32(self.num)
writer.writeUInt32(0)
for offsets in self.offsets:
writer.writeUInt32(offsets)
size = writer.tell() - start
with writer.seek(sizeofs):
writer.writeUInt32(size)
return writer
def save(self, writer=None):
if writer is None:
writer = BinaryIO()
start = writer.tell()
writer.write(self.magic)
headersizeofs = writer.tell()
writer.writeUInt32(0)
writer.writeUInt32(0)
writer.writeUInt32(0)
writer.writeUInt32(0)
sizeofs = writer.tell()
writer.writeUInt32(0)
writer.writeUInt32(0)
size = writer.tell() - start
with writer.seek(headersizeofs):
writer.writeUInt32(size)
writer = self.fato.save(writer)
writer = self.fatb.save(writer)
writer = self.fimb.save(writer)
with writer.seek(sizeofs):
writer.writeUInt32(size)
return writer
def save(self, writer=None):
writer = BinaryIO(writer)
start = writer.tell()
if not self.compiled_scripts and self.scripts:
handle = BinaryIO()
self.export(handle)
handle.seek(0)
self.import_(handle)
blocks = [] # set(self.engine.blocks)
for block in self.engine.blocks:
for used_block in blocks:
if block == used_block:
break
else:
blocks.append(block)
for block in self.compiled_scripts:
writer.writeUInt32(0)
writer.writeUInt32(0xFD13)
for block in blocks:
block.offset = writer.tell()-start
writer.write(block.buff)
writer.writeAlign(4)
for block in blocks:
for ofs, dest in block.jumps.items():
for used_block in blocks:
if used_block == dest:
dest = used_block
with writer.seek(start+block.offset+ofs):
writer.writeInt32(dest.offset-block.offset-ofs-4)
with writer.seek(start):
for block in self.compiled_scripts:
# there is a chance that a block in the scripts got removed
# use the actual written blocks for the relevant offsets
for used_block in blocks:
if used_block == block:
writer.writeInt32(start+used_block.offset
- writer.tell()-4)
break
return writer
def load(self, reader):
self.offsets = []
self.scripts = []
self.functions = []
self.func_map = {} # {offset: [func, count]}
reader = BinaryIO.reader(reader)
start = reader.tell()
try:
offset = reader.readUInt32()
except:
# Empty File. No script contents
return
while offset:
abs_offset = offset+reader.tell()
current_pos = reader.tell()
for offset in self.offsets:
if current_pos > offset:
break
self.offsets.append(abs_offset)
try:
offset = reader.readUInt32()
except:
# Exhaustive offset list: not a script
return
if offset & 0xFFFF == 0xFD13:
break
if not self.offsets:
return
for scrnum, offset in enumerate(self.offsets, self.script_start):
with reader.seek(offset):
script = ScriptDecompiler(reader, self)
script.parse()
script.header_lines.append('def script_{num}(engine):'
.format(num=scrnum))
self.scripts.append(script)
changed = True
while changed:
changed = False
for offset, (func, count, func_id) in self.func_map.items():
if func is None:
changed = True
with reader.seek(offset):
script = ScriptDecompiler(reader, self)
script.parse()
self.func_map[offset][0] = script
cur_id = self.function_start
embedded_functions = []
for (offset, (func, count, func_id)) in self.func_map.items():
if count > 1:
func.header_lines.append('def func_{num}(engine):'
.format(num=cur_id))
self.functions.append(func)
self.func_map[offset][2] = cur_id
cur_id += 1
else:
embedded_functions.append(func)
for script in itertools.chain(self.scripts, self.functions,
embedded_functions):
for expr in script:
try:
if expr.target.args[0].name != 'jump':
continue
except:
continue
offset = expr.target.args[0].args[0]
func, count, func_id = self.func_map[offset]
if func_id is None:
func.indent = 0
else:
end = func.lines[-1]
func = script.func(
'call', 'func_{0}'.format(func_id),
namespace='engine.')
if end.is_return() and end.args and end.args != (None, ):
func = script.end(func)
expr.set_target(func)
def save(self, writer=None):
if writer is None:
writer = BinaryIO()
start = writer.tell()
writer.write(self.magic)
sizeofs = writer.tell()
writer.writeUInt32(0)
data = []
for fdata, entry in zip(self.files, self.narc.fatb.entries):
total = len(data)
if entry.start > total:
data += '\x00' * (entry.start - total)
data[entry] = fdata
writer.write(''.join(data))
size = writer.tell() - start
with writer.seek(sizeofs):
writer.writeUInt32(size)
return writer
def save(self, writer=None):
writer = writer if writer is not None else BinaryIO()
start = writer.tell()
writer.write(self.magic)
size_ofs = writer.tell()
writer.writeUInt32(0)
writer = self.texinfo.save(writer)
writer = self.tex4x4info.save(writer)
writer = self.palinfo.save(writer)
writer.writeAlign(4)
ofs = writer.tell() - start
with writer.seek(self.texinfo._lookupofs_ofs):
writer.writeUInt16(ofs)
for i in xrange(self.texdict.num):
texparam = self.texparams[i]
self.texdict.data[i] = \
struct.pack('II',
(texparam.ofs >> 3) |
((log2(texparam.width >> 3)) << 20) |
((log2(texparam.height >> 3)) << 23) |
(texparam.format << 26) |
(texparam.color0 << 29), 0)
writer = self.texdict.save(writer)
writer.writeAlign(4)
ofs = writer.tell() - start
with writer.seek(self.palinfo._lookupofs_ofs):
writer.writeUInt16(ofs)
for i in xrange(self.paldict.num):
param = self.palparams[i]
self.paldict.data[i] = struct.pack('HH', param.ofs >> 3,
param.count4)
writer = self.paldict.save(writer)
writer.writeAlign(8)
ofs = writer.tell() - start
with writer.seek(self.texinfo._dataofs_ofs):
writer.writeUInt32(ofs) # texinfo dataofs
datastart = writer.tell()
writer.write(self.texdata)
writer.writeAlign(8)
size = writer.tell() - datastart
with writer.seek(self.texinfo._datasize_ofs):
writer.writeUInt16(size >> 3) # texinfo datasize
writer.writeAlign(8)
ofs = writer.tell() - start
with writer.seek(self.palinfo._dataofs_ofs):
writer.writeUInt32(ofs) # palinfo dataofs
datastart = writer.tell()
writer.write(self.paldata)
writer.writeAlign(8)
size = writer.tell() - datastart
with writer.seek(self.palinfo._datasize_ofs):
writer.writeUInt16(size >> 3) # palinfo datasize
size = writer.tell() - start
with writer.seek(size_ofs):
writer.writeUInt32(size)
return writer
def save(self, writer=None):
if writer is None:
writer = BinaryIO()
start = writer.tell()
writer.write(self.magic)
sizeofs = writer.tell()
writer.writeUInt32(0)
writer.writeUInt16(self.num)
writer.writeUInt16(0)
for entry in self.entries:
writer.writeUInt32(entry.start)
writer.writeUInt32(entry.stop)
size = writer.tell() - start
with writer.seek(sizeofs):
writer.writeUInt32(size)
return writer
def save(self, writer=None):
if writer is None:
writer = BinaryIO()
start = writer.tell()
writer.write(self.magic)
writer.writeUInt16(self.endian)
writer.writeUInt16(self.version)
sizeofs = writer.tell()
writer.writeUInt32(0)
headersizeofs = writer.tell()
writer.writeUInt16(0)
writer.writeUInt16(self.numblocks)
size = writer.tell() - start
with writer.seek(headersizeofs):
writer.writeUInt16(size)
writer = self.fatb.save(writer)
writer = self.fntb.save(writer)
writer = self.fimg.save(writer)
writer.writeUInt32(0)
size = writer.tell() - start
with writer.seek(sizeofs):
writer.writeUInt32(size)
return writer
def load(self, reader):
reader = BinaryIO.reader(reader)
AtomicStruct.load(self, reader)
data = reader.read(self.size_ - 8)
self.files.extend([data[entry] for entry in self.narc.fatb.entries_])
def load(self, reader):
reader = BinaryIO.reader(reader)
AtomicStruct.load(self, reader)
for i in xrange(self._data.num):
self.entries_.append(
slice(reader.readUInt32(), reader.readUInt32()))
def load(self, reader):
reader = BinaryIO.reader(reader)
AtomicStruct.load(self, reader)
self.fatb.load(reader)
self.fntb.load(reader)
self.fimg.load(reader)