def load_ifs(self, path):
self.is_file = True
name = basename(path)
self.ifs_out = name
self.folder_out = splitext(name)[0] + '_ifs'
self.default_out = self.folder_out
self.file = open(path, 'rb')
header = ByteBuffer(self.file.read(36))
signature = header.get_u32()
if signature != SIGNATURE:
raise IOError('Given file was not an IFS file!')
self.file_version = header.get_u16()
# next u16 is just NOT(version)
assert header.get_u16() ^ self.file_version == 0xFFFF
self.time = header.get_u32()
ifs_tree_size = header.get_u32()
manifest_end = header.get_u32()
self.data_blob = FileBlob(self.file, manifest_end)
if self.file_version > 1:
# md5 of manifest, unchecked
header.offset += 16
self.file.seek(header.offset)
self.manifest = KBinXML(self.file.read(manifest_end - header.offset))
self.tree = GenericFolder(self.data_blob, self.manifest.xml_doc)
def load_ifs(self, path):
self.is_file = True
name = basename(path)
self.ifs_out = name
self.folder_out = splitext(name)[0] + '_ifs'
self.default_out = self.folder_out
with open(path, 'rb') as f:
file = ByteBuffer(f.read())
signature = file.get_u32()
if signature != SIGNATURE:
raise IOError('Given file was not an IFS file!')
self.file_version = file.get_u16()
# next u16 is just NOT(version)
assert file.get_u16() ^ self.file_version == 0xFFFF
self.time = file.get_u32()
ifs_tree_size = file.get_u32()
manifest_end = file.get_u32()
self.data_blob = bytes(file.data[manifest_end:])
# 16 bytes for manifest md5, unchecked
self.manifest = KBinXML(file.data[HEADER_SIZE:])
self.tree = GenericFolder(self.data_blob, self.manifest.xml_doc)
assert ifs_tree_size == self.tree_size
def _load_from_ifs(self, convert_kbin=True, **kwargs):
data = self.ifs_data.get(self.start, self.size)
if convert_kbin and self.name.endswith(
'.xml') and KBinXML.is_binary_xml(data):
data = KBinXML(data).to_text().encode('utf8')
return data
def _load_from_filesystem(self):
raw = GenericFile._load_from_filesystem(self)
k = KBinXML(raw)
# force the only type we can compress
for tex in k.xml_doc.iterchildren():
tex.attrib['format'] = 'argb8888rev'
return k.to_binary()
def _load_from_ifs(self, convert_kbin=True):
end = self.start + self.size
assert self.start <= len(self.ifs_data) and end <= len(self.ifs_data)
data = self.ifs_data[self.start:end]
if convert_kbin and self.name.endswith(
'.xml') and KBinXML.is_binary_xml(data):
data = KBinXML(data).to_text().encode('utf8')
return data
def _load_from_filesystem(self):
raw = GenericFile._load_from_filesystem(self)
k = KBinXML(raw)
# fallback to a type we can encode
for tex in k.xml_doc.iterchildren():
if tex.attrib['format'] not in cachable_formats:
tex.attrib['format'] = 'argb8888rev'
return k.to_binary()
def get_raw_xml(input):
try:
output = KBinXML(input).to_text()
root = etree.fromstring(output.encode('utf-8'))
return etree.tostring(root,
encoding='unicode',
pretty_print=True)
except:
return ""
def repack(self, manifest, data_blob, tqdm_progress):
if tqdm_progress:
tqdm_progress.write(self.full_path)
tqdm_progress.update(1)
elem = etree.SubElement(manifest, self.packed_name)
elem.attrib['__type'] = '3s32'
data = self.load(convert_kbin=False)
if self.name.endswith('.xml') and not KBinXML.is_binary_xml(data):
data = KBinXML(data).to_binary()
# offset, size, timestamp
elem.text = '{} {} {}'.format(len(data_blob.getvalue()), len(data),
self.time)
data_blob.write(data)
def repack(self, progress=True, path=None, **kwargs):
if path is None:
path = self.ifs_out
# open first in case path is bad
ifs_file = open(path, 'wb')
self.data_blob = BytesIO()
self.manifest = KBinXML(etree.Element('imgfs'))
manifest_info = etree.SubElement(self.manifest.xml_doc, '_info_')
# the important bit
data = self._repack_tree(progress, **kwargs)
data_md5 = etree.SubElement(manifest_info, 'md5')
data_md5.attrib['__type'] = 'bin'
data_md5.attrib['__size'] = '16'
data_md5.text = hashlib.md5(data).hexdigest()
data_size = etree.SubElement(manifest_info, 'size')
data_size.attrib['__type'] = 'u32'
data_size.text = str(len(data))
manifest_bin = self.manifest.to_binary()
manifest_hash = hashlib.md5(manifest_bin).digest()
head = ByteBuffer()
head.append_u32(SIGNATURE)
head.append_u16(self.file_version)
head.append_u16(self.file_version ^ 0xFFFF)
head.append_u32(int(unixtime()))
head.append_u32(self.manifest.mem_size)
manifest_end = len(manifest_bin) + head.offset + 4
if self.file_version > 1:
manifest_end += 16
head.append_u32(manifest_end)
if self.file_version > 1:
head.append_bytes(manifest_hash)
ifs_file.write(head.data)
ifs_file.write(manifest_bin)
ifs_file.write(data)
ifs_file.close()
def tree_complete(self):
GenericFolder.tree_complete(self)
for filename, file in self.files.items():
if filename.endswith('.xml'):
self.info_file = file
break
if not self.info_file:
raise KeyError('MD5 folder contents have no mapping xml')
self.info_kbin = KBinXML(self.info_file.load(convert_kbin = False))
self._apply_md5()
def load_ifs(self,
path,
super_disable=False,
super_skip_bad=False,
super_abort_if_bad=False):
self.is_file = True
name = basename(path)
self.ifs_out = name
self.folder_out = splitext(name)[0] + '_ifs'
self.default_out = self.folder_out
self.file = open(path, 'rb')
header = ByteBuffer(self.file.read(36))
signature = header.get_u32()
if signature != SIGNATURE:
raise IOError('Given file was not an IFS file!')
self.file_version = header.get_u16()
# next u16 is just NOT(version)
assert header.get_u16() ^ self.file_version == 0xFFFF
self.time = header.get_u32()
ifs_tree_size = header.get_u32()
manifest_end = header.get_u32()
self.data_blob = FileBlob(self.file, manifest_end)
self.manifest_md5 = None
if self.file_version > 1:
self.manifest_md5 = header.get_bytes(16)
self.file.seek(header.offset)
self.manifest = KBinXML(self.file.read(manifest_end - header.offset))
self.tree = GenericFolder(self.data_blob,
self.manifest.xml_doc,
super_disable=super_disable,
super_skip_bad=super_skip_bad,
super_abort_if_bad=super_abort_if_bad)
def tree_complete(self):
GenericFolder.tree_complete(self)
self.info_kbin = None
self.info_file = None
for filename, file in self.files.items():
if filename.endswith('.xml'):
self.info_file = file
break
if not self.info_file:
#raise KeyError('MD5 folder contents have no mapping xml')
# _super_ references to info XML breaks things - just extract what we can
return
self.info_kbin = KBinXML(self.info_file.load(convert_kbin=False))
self._apply_md5()
def read(self) -> etree:
# Lets grab the raw data from the request
xml_bin = self._request.data
# Decrypt the data if necessary
x_eamuse_info: Optional[str] = None
if self.encrypted:
x_eamuse_info, key = self._get_encryption_data()
xml_bin = EAmuseARC4(key).decrypt(xml_bin)
# De-Compress the data if necessary
# Right now we only support `lz77`
if self.compressed and self.compression == "lz77":
xml_bin = Lz77().decompress(xml_bin)
# Convert the binary xml data to text bytes, and save a copy
xml_bytes = KBinXML(xml_bin).to_text().encode(self.ENCODING)
self._save_xml(xml_bytes, "req", x_eamuse_info)
# Convert the XML text to an eTree
xml_root = etree.fromstring(xml_bytes)
# Grab the common xml information that we need
# <call model="_MODEL_" srcid="_SRCID_">
# <_MODULE_ method="_METHOD_" command="_COMMAND_">
# </call>
# First grab the model
self.model = Model.from_modelstring(get_xml_attrib(xml_root, "model"))
# The module is the first child of the root
module = xml_root[0]
self.module = get_xml_tag(module)
self.method = get_xml_attrib(module, "method")
self.command = get_xml_attrib(module, "command")
rlogger.info(self.__repr__())
rlogger.debug(f"Request:\n {xml_bytes.decode(self.ENCODING)}")
return xml_root
def __decode(self, data: bytes) -> Node:
"""
Given data, decode the data into a Node tree.
Parameters:
data - Binary string representing data to decode.
Returns:
Node tree on success or None on failure.
"""
# Assume it's a binary page
binary = BinaryEncoding()
ret = binary.decode(data, skip_on_exceptions=True)
if ret is not None:
# We got a result, it was binary
self.last_text_encoding = binary.encoding
self.last_packet_encoding = EAmuseProtocol.BINARY
return ret
try:
data = KBinXML(data)
except:
pass
# Assume its XML
xml = XmlEncoding()
ret = xml.decode(data, skip_on_exceptions=True)
if ret is not None:
# We got a result, it was XML
self.last_text_encoding = xml.encoding
self.last_packet_encoding = EAmuseProtocol.XML
return ret
# Couldn't decode
raise EAmuseException("Unknown packet encoding")
def response(self, xml_bytes: Union[bytes, eElement]):
# Firstly, let's make sure xml_bytes is a bytes object
if type(xml_bytes) == eElement:
xml_bytes = etree.tostring(xml_bytes, pretty_print=True)
# Generate our own encryption key
x_eamuse_info, key = self._make_encryption_data()
# Save our xml response
self._save_xml(xml_bytes, "resp",
x_eamuse_info if self.encrypted else None)
# Convert our xml to binary
xml_bin = KBinXML(xml_bytes).to_binary()
# Common Headers
headers = {
"Content-Type": "application/octet_stream",
"Server": "Microsoft-HTTPAPI/2.0",
}
# Compress the data if necessary
# Right now we only support `lz77`
if self.compressed and self.compression == "lz77":
xml_bin = Lz77().compress(xml_bin)
headers[self.X_COMPRESS] = "lz77"
# Encrypt the data if necessary
if self.encrypted:
xml_bin = EAmuseARC4(key).encrypt(xml_bin)
headers[self.X_EAMUSE_INFO] = x_eamuse_info
rlogger.debug(f"Response:\n{xml_bytes.decode(self.ENCODING)}")
# Update the Static Request ID
ServiceRequest.REQUEST_ID += 1
return xml_bin, headers
def main():
for subdir in ['launcher', 'updater', 'game']:
enc_path = os.path.join(enc_base, subdir)
dec_path = os.path.join(dec_base, subdir)
mountfile = os.path.join(enc_path, 'conf', 'mounttable.binr')
configfile = os.path.join(enc_path, 'conf', 'config.binr')
# whatever
with open(mountfile, 'rb') as f:
mounts = decrypt(mountfile, f.read())
mounts = KBinXML(mounts)
with open(configfile, 'rb') as f:
config = decrypt(configfile, f.read())
config = KBinXML(config)
cfg_dec = os.path.join(dec_path, 'config')
mkdir_p(cfg_dec)
with open(os.path.join(cfg_dec, 'config.xml'), 'w') as f:
f.write(config.to_text())
with open(os.path.join(cfg_dec, 'mounttable.xml'), 'w') as f:
f.write(mounts.to_text())
for vfs in tqdm(mounts.xml_doc):
dst = vfs.attrib['dst']
src = vfs.attrib['src']
src = src.lstrip('/')
dst = dst.lstrip('/')
src = os.path.join(enc_path, src)
dst = os.path.join(dec_path, dst)
if dst[-1].isdigit():
dst = dst[:-1]
ifs = IFS(src)
for f in ifs.tree.all_files:
if f.name.endswith('r') and type(f) == GenericFile and type(
f.parent) == GenericFolder:
f.__class__ = CryptFile
f.name = f.name[:-1] # strip r
mkdir_p(dst)
ifs.extract(use_cache=False, recurse=False, path=dst)
def get_binxml(input):
return KBinXML(input.encode('utf-8')).to_binary()
class IFS:
def __init__(self, path):
if isfile(path):
self.load_ifs(path)
elif isdir(path):
self.load_dir(path)
else:
raise IOError('Input path does not exist')
def load_ifs(self, path):
self.is_file = True
name = basename(path)
self.ifs_out = name
self.folder_out = splitext(name)[0] + '_ifs'
self.default_out = self.folder_out
with open(path, 'rb') as f:
file = ByteBuffer(f.read())
signature = file.get_u32()
if signature != SIGNATURE:
raise IOError('Given file was not an IFS file!')
self.file_version = file.get_u16()
# next u16 is just NOT(version)
assert file.get_u16() ^ self.file_version == 0xFFFF
self.time = file.get_u32()
ifs_tree_size = file.get_u32()
manifest_end = file.get_u32()
self.data_blob = bytes(file.data[manifest_end:])
# 16 bytes for manifest md5, unchecked
self.manifest = KBinXML(file.data[HEADER_SIZE:])
self.tree = GenericFolder(self.data_blob, self.manifest.xml_doc)
assert ifs_tree_size == self.tree_size
def load_dir(self, path):
self.is_file = False
path = path.rstrip('/\\')
self.folder_out = basename(path)
if '_ifs' in self.folder_out:
self.ifs_out = self.folder_out.replace('_ifs', '.ifs')
else:
self.ifs_out = self.folder_out + '.ifs'
self.default_out = self.ifs_out
self.file_version = FILE_VERSION
self.time = int(getmtime(path))
self.data_blob = None
self.manifest = None
os_tree = self._create_dir_tree(path)
self.tree = GenericFolder(None, os_tree)
def _create_dir_tree(self, path):
tree = self._create_dir_tree_recurse(walk(path))
if 'ifs_manifest.xml' in tree['files']:
tree['files'].remove('ifs_manifest.xml')
return tree
def _create_dir_tree_recurse(self, walker):
tree = {}
root, dirs, files = next(walker)
tree['path'] = root
tree['files'] = files
tree['folders'] = []
for dir in dirs:
tree['folders'].append(self._create_dir_tree_recurse(walker))
return tree
def __str__(self):
return str(self.tree)
def extract(self, progress = True, use_cache = True, recurse = True, tex_only = False, path = None):
if path is None:
path = self.folder_out
if tex_only and 'tex' not in self.tree.folders:
return
utils.mkdir_silent(path)
utime(path, (self.time, self.time))
if self.manifest and not tex_only:
with open(join(path, 'ifs_manifest.xml'), 'wb') as f:
f.write(self.manifest.to_text().encode('utf8'))
# build the tree
for folder in self.tree.all_folders:
if tex_only and folder.name != 'tex':
continue
f_path = join(path, folder.full_path)
utils.mkdir_silent(f_path)
utime(f_path, (self.time, self.time))
# extract the files
for f in tqdm(self.tree.all_files):
if tex_only and not isinstance(f, ImageFile):
continue
f.extract(path, use_cache)
if progress:
tqdm.write(f.full_path)
if recurse and f.name.endswith('.ifs'):
rpath = join(path, f.full_path)
i = IFS(rpath)
i.extract(progress, use_cache, recurse, rpath.replace('.ifs','_ifs'))
''' If you can get shared memory for IFS.data_blob working, this will
be a lot faster. As it is, it gets pickled for every file, and
is 3x slower than the serial implementation even with image extraction
'''
# extract the tree
'''p = Pool()
args = zip(self.tree.all_files, itertools.cycle((path,)))
for f in tqdm(p.imap_unordered(_extract, args)):
if progress:
tqdm.write(f)'''
def repack(self, progress = True, use_cache = True, path = None):
if path is None:
path = self.ifs_out
# open first in case path is bad
ifs_file = open(path, 'wb')
self.data_blob = BytesIO()
self.manifest = KBinXML(etree.Element('imgfs'))
manifest_info = etree.SubElement(self.manifest.xml_doc, '_info_')
# the important bit
data = self._repack_tree(progress, use_cache)
data_md5 = etree.SubElement(manifest_info, 'md5')
data_md5.attrib['__type'] = 'bin'
data_md5.attrib['__size'] = '16'
data_md5.text = hashlib.md5(data).hexdigest()
data_size = etree.SubElement(manifest_info, 'size')
data_size.attrib['__type'] = 'u32'
data_size.text = str(len(data))
manifest_bin = self.manifest.to_binary()
manifest_end = HEADER_SIZE + len(manifest_bin)
manifest_hash = hashlib.md5(manifest_bin).digest()
head = ByteBuffer()
head.append_u32(SIGNATURE)
head.append_u16(self.file_version)
head.append_u16(self.file_version ^ 0xFFFF)
head.append_u32(int(unixtime()))
head.append_u32(self.tree_size)
head.append_u32(manifest_end)
ifs_file.write(head.data)
ifs_file.write(manifest_hash)
ifs_file.write(manifest_bin)
ifs_file.write(data)
ifs_file.close()
def _repack_tree(self, progress = True, use_cache = True):
folders = self.tree.all_folders
files = self.tree.all_files
# Can't pickle lmxl, so to dirty-hack land we go
kbin_backup = []
for folder in folders:
if isinstance(folder, MD5Folder):
kbin_backup.append(folder.info_kbin)
folder.info_kbin = None
needs_preload = (f for f in files if f.needs_preload or not use_cache)
args = list(zip(needs_preload, itertools.cycle((use_cache,))))
p = Pool()
for f in tqdm(p.imap_unordered(_load, args), desc='Caching', total=len(args), disable = not progress):
if progress:
tqdm.write(f)
# restore stuff from before
for folder in folders:
if isinstance(folder, MD5Folder):
folder.info_kbin = kbin_backup.pop(0)
tqdm_progress = None
if progress:
tqdm_progress = tqdm(desc='Writing', total=len(files))
self.tree.repack(self.manifest.xml_doc, self.data_blob, tqdm_progress)
return self.data_blob.getvalue()
# suspected to be the in-memory representation
@property
def tree_size(self):
BASE_SIZE = 856
return BASE_SIZE + self._tree_size_recurse(self.tree)
def _tree_size_recurse(self, tree, depth = 0):
FILE = 64
FOLDER = 56
DEPTH_MULTIPLIER = 16
size = len(tree.files) * FILE
size += len(tree.folders) * (FOLDER - depth*DEPTH_MULTIPLIER)
for name, folder in tree.folders.items():
size += self._tree_size_recurse(folder, depth+1)
return size
class IFS:
def __init__(self,
path,
super_disable=False,
super_skip_bad=False,
super_abort_if_bad=False):
if isfile(path):
self.load_ifs(path, super_disable, super_skip_bad,
super_abort_if_bad)
elif isdir(path):
self.load_dir(path)
else:
raise IOError('Input path {} does not exist'.format(path))
def load_ifs(self,
path,
super_disable=False,
super_skip_bad=False,
super_abort_if_bad=False):
self.is_file = True
name = basename(path)
self.ifs_out = name
self.folder_out = splitext(name)[0] + '_ifs'
self.default_out = self.folder_out
self.file = open(path, 'rb')
header = ByteBuffer(self.file.read(36))
signature = header.get_u32()
if signature != SIGNATURE:
raise IOError('Given file was not an IFS file!')
self.file_version = header.get_u16()
# next u16 is just NOT(version)
assert header.get_u16() ^ self.file_version == 0xFFFF
self.time = header.get_u32()
ifs_tree_size = header.get_u32()
manifest_end = header.get_u32()
self.data_blob = FileBlob(self.file, manifest_end)
self.manifest_md5 = None
if self.file_version > 1:
self.manifest_md5 = header.get_bytes(16)
self.file.seek(header.offset)
self.manifest = KBinXML(self.file.read(manifest_end - header.offset))
self.tree = GenericFolder(self.data_blob,
self.manifest.xml_doc,
super_disable=super_disable,
super_skip_bad=super_skip_bad,
super_abort_if_bad=super_abort_if_bad)
# IFS files repacked with other tools usually have wrong values - don't validate this
#assert ifs_tree_size == self.manifest.mem_size
def load_dir(self, path):
self.is_file = False
self.file = None
path = path.rstrip('/\\')
self.folder_out = basename(path)
if '_ifs' in self.folder_out:
self.ifs_out = self.folder_out.replace('_ifs', '.ifs')
else:
self.ifs_out = self.folder_out + '.ifs'
self.default_out = self.ifs_out
self.file_version = FILE_VERSION
self.time = int(getmtime(path))
self.data_blob = None
self.manifest = None
os_tree = self._create_dir_tree(path)
self.tree = GenericFolder(None, os_tree)
def _create_dir_tree(self, path):
tree = self._create_dir_tree_recurse(walk(path))
if 'ifs_manifest.xml' in tree['files']:
tree['files'].remove('ifs_manifest.xml')
return tree
def _create_dir_tree_recurse(self, walker):
tree = {}
root, dirs, files = next(walker)
tree['path'] = root
tree['files'] = files
tree['folders'] = []
for dir in dirs:
tree['folders'].append(self._create_dir_tree_recurse(walker))
return tree
def close(self):
if self.file:
self.file.close()
def __str__(self):
return str(self.tree)
def extract(self,
progress=True,
recurse=True,
tex_only=False,
extract_manifest=False,
path=None,
rename_dupes=False,
**kwargs):
if path is None:
path = self.folder_out
if tex_only:
kwargs['use_cache'] = False
utils.mkdir_silent(path)
utime(path, (self.time, self.time))
if extract_manifest and self.manifest and not tex_only:
with open(join(path, 'ifs_manifest.xml'), 'wb') as f:
f.write(self.manifest.to_text().encode('utf8'))
# build the tree
for folder in self.tree.all_folders:
if tex_only and folder.name == 'tex':
self.tree = folder
# make it root to discourage repacking
folder.name = ''
for f in folder.all_files:
f.path = ''
break
elif tex_only:
continue
f_path = join(path, folder.full_path)
utils.mkdir_silent(f_path)
utime(f_path, (self.time, self.time))
# handle different-case-but-same-name for Windows
same_name = defaultdict(list)
for name, obj in folder.files.items():
same_name[name.lower()].append(obj)
for files in same_name.values():
# common base case of "sane ifs file"
if len(files) == 1:
continue
# make them 'a (1)', 'a (2)' etc
if rename_dupes:
for i, f in enumerate(files[1:]):
base, ext = splitext(f.name)
f.name = base + ' ({})'.format(i + 1) + ext
elif progress: # warn if not silenced
all_names = ', '.join([f.name for f in files])
tqdm.write(
'WARNING: Files with same name and differing case will overwrite on Windows ({}). '
.format(all_names) +
'Use --rename-dupes to extract without loss')
# else just do nothing
# extract the files
for f in tqdm(self.tree.all_files, disable=not progress):
# allow recurse + tex_only to extract ifs files
if tex_only and not isinstance(f, ImageFile) and not isinstance(
f, ImageCanvas) and not (recurse
and f.name.endswith('.ifs')):
continue
f.extract(path, **kwargs)
if progress:
tqdm.write(f.full_path)
if recurse and f.name.endswith('.ifs'):
rpath = join(path, f.full_path)
i = IFS(rpath)
i.extract(progress=progress,
recurse=recurse,
tex_only=tex_only,
extract_manifest=extract_manifest,
path=rpath.replace('.ifs', '_ifs'),
rename_dupes=rename_dupes,
**kwargs)
# you can't pickle open files, so this won't work. Perhaps there is a way around it?
'''to_extract = (f for f in self.tree.all_files if not(tex_only and not isinstance(f, ImageFile) and not isinstance(f, ImageCanvas)))
p = Pool()
args = zip(to_extract, itertools.cycle((path,)), itertools.cycle((kwargs,)))
to_recurse = []
for f in tqdm(p.imap_unordered(_extract, args)):
if progress:
tqdm.write(f)
if recurse and f.name.endswith('.ifs'):
to_recurse.append(join(path, f.full_path))
for rpath in recurse:
i = IFS(rpath)
i.extract(progress=progress, recurse=recurse, tex_only=tex_only,
extract_manifest=extract_manifest, path=rpath.replace('.ifs','_ifs'), **kwargs)'''
def repack(self, progress=True, path=None, **kwargs):
if path is None:
path = self.ifs_out
# open first in case path is bad
ifs_file = open(path, 'wb')
self.data_blob = BytesIO()
self.manifest = KBinXML(etree.Element('imgfs'))
manifest_info = etree.SubElement(self.manifest.xml_doc, '_info_')
# the important bit
data = self._repack_tree(progress, **kwargs)
data_md5 = etree.SubElement(manifest_info, 'md5')
data_md5.attrib['__type'] = 'bin'
data_md5.attrib['__size'] = '16'
data_md5.text = hashlib.md5(data).hexdigest()
data_size = etree.SubElement(manifest_info, 'size')
data_size.attrib['__type'] = 'u32'
data_size.text = str(len(data))
manifest_bin = self.manifest.to_binary()
manifest_hash = hashlib.md5(manifest_bin).digest()
head = ByteBuffer()
head.append_u32(SIGNATURE)
head.append_u16(self.file_version)
head.append_u16(self.file_version ^ 0xFFFF)
head.append_u32(int(unixtime()))
head.append_u32(self.manifest.mem_size)
manifest_end = len(manifest_bin) + head.offset + 4
if self.file_version > 1:
manifest_end += 16
head.append_u32(manifest_end)
if self.file_version > 1:
head.append_bytes(manifest_hash)
ifs_file.write(head.data)
ifs_file.write(manifest_bin)
ifs_file.write(data)
ifs_file.close()
def _repack_tree(self, progress=True, **kwargs):
folders = self.tree.all_folders
files = self.tree.all_files
# Can't pickle lmxl, so to dirty-hack land we go
kbin_backup = []
for folder in folders:
if isinstance(folder, MD5Folder):
kbin_backup.append(folder.info_kbin)
folder.info_kbin = None
needs_preload = (f for f in files
if f.needs_preload or not kwargs['use_cache'])
args = list(zip(needs_preload, itertools.cycle((kwargs, ))))
p = Pool()
for f in tqdm(p.imap_unordered(_load, args),
desc='Caching',
total=len(args),
disable=not progress):
if progress:
tqdm.write(f)
p.close()
p.terminate()
# restore stuff from before
for folder in folders:
if isinstance(folder, MD5Folder):
folder.info_kbin = kbin_backup.pop(0)
tqdm_progress = None
if progress:
tqdm_progress = tqdm(desc='Writing', total=len(files))
self.tree.repack(self.manifest.xml_doc, self.data_blob, tqdm_progress,
**kwargs)
return self.data_blob.getvalue()
class IFS:
def __init__(self, path):
if isfile(path):
self.load_ifs(path)
elif isdir(path):
self.load_dir(path)
else:
raise IOError('Input path {} does not exist'.format(path))
def load_ifs(self, path):
self.is_file = True
name = basename(path)
self.ifs_out = name
self.folder_out = splitext(name)[0] + '_ifs'
self.default_out = self.folder_out
self.file = open(path, 'rb')
header = ByteBuffer(self.file.read(36))
signature = header.get_u32()
if signature != SIGNATURE:
raise IOError('Given file was not an IFS file!')
self.file_version = header.get_u16()
# next u16 is just NOT(version)
assert header.get_u16() ^ self.file_version == 0xFFFF
self.time = header.get_u32()
ifs_tree_size = header.get_u32()
manifest_end = header.get_u32()
self.data_blob = FileBlob(self.file, manifest_end)
if self.file_version > 1:
# md5 of manifest, unchecked
header.offset += 16
self.file.seek(header.offset)
self.manifest = KBinXML(self.file.read(manifest_end - header.offset))
self.tree = GenericFolder(self.data_blob, self.manifest.xml_doc)
# IFS files repacked with other tools usually have wrong values - don't validate this
#assert ifs_tree_size == self.manifest.mem_size
def load_dir(self, path):
self.is_file = False
path = path.rstrip('/\\')
self.folder_out = basename(path)
if '_ifs' in self.folder_out:
self.ifs_out = self.folder_out.replace('_ifs', '.ifs')
else:
self.ifs_out = self.folder_out + '.ifs'
self.default_out = self.ifs_out
self.file_version = FILE_VERSION
self.time = int(getmtime(path))
self.data_blob = None
self.manifest = None
os_tree = self._create_dir_tree(path)
self.tree = GenericFolder(None, os_tree)
def _create_dir_tree(self, path):
tree = self._create_dir_tree_recurse(walk(path))
if 'ifs_manifest.xml' in tree['files']:
tree['files'].remove('ifs_manifest.xml')
return tree
def _create_dir_tree_recurse(self, walker):
tree = {}
root, dirs, files = next(walker)
tree['path'] = root
tree['files'] = files
tree['folders'] = []
for dir in dirs:
tree['folders'].append(self._create_dir_tree_recurse(walker))
return tree
def close(self):
if self.file:
self.file.close()
def __str__(self):
return str(self.tree)
def extract(self,
progress=True,
recurse=True,
tex_only=False,
extract_manifest=False,
path=None,
**kwargs):
if path is None:
path = self.folder_out
if tex_only:
kwargs['use_cache'] = False
if 'tex' not in self.tree.folders:
return
utils.mkdir_silent(path)
utime(path, (self.time, self.time))
if extract_manifest and self.manifest and not tex_only:
with open(join(path, 'ifs_manifest.xml'), 'wb') as f:
f.write(self.manifest.to_text().encode('utf8'))
# build the tree
for folder in self.tree.all_folders:
if tex_only and folder.name == 'tex':
self.tree = folder
# make it root to discourage repacking
folder.name = ''
for f in folder.all_files:
f.path = ''
break
elif tex_only:
continue
f_path = join(path, folder.full_path)
utils.mkdir_silent(f_path)
utime(f_path, (self.time, self.time))
# extract the files
for f in tqdm(self.tree.all_files):
if tex_only and not isinstance(f, ImageFile) and not isinstance(
f, ImageCanvas):
continue
f.extract(path, **kwargs)
if progress:
tqdm.write(f.full_path)
if recurse and f.name.endswith('.ifs'):
rpath = join(path, f.full_path)
i = IFS(rpath)
i.extract(progress=progress,
recurse=recurse,
tex_only=tex_only,
extract_manifest=extract_manifest,
path=rpath.replace('.ifs', '_ifs'),
**kwargs)
''' Todo: reimplement this since we're using file objects now
'''
# extract the tree
'''p = Pool()
args = zip(self.tree.all_files, itertools.cycle((path,)))
for f in tqdm(p.imap_unordered(_extract, args)):
if progress:
tqdm.write(f)'''
def repack(self, progress=True, path=None, **kwargs):
if path is None:
path = self.ifs_out
# open first in case path is bad
ifs_file = open(path, 'wb')
self.data_blob = BytesIO()
self.manifest = KBinXML(etree.Element('imgfs'))
manifest_info = etree.SubElement(self.manifest.xml_doc, '_info_')
# the important bit
data = self._repack_tree(progress, **kwargs)
data_md5 = etree.SubElement(manifest_info, 'md5')
data_md5.attrib['__type'] = 'bin'
data_md5.attrib['__size'] = '16'
data_md5.text = hashlib.md5(data).hexdigest()
data_size = etree.SubElement(manifest_info, 'size')
data_size.attrib['__type'] = 'u32'
data_size.text = str(len(data))
manifest_bin = self.manifest.to_binary()
manifest_hash = hashlib.md5(manifest_bin).digest()
head = ByteBuffer()
head.append_u32(SIGNATURE)
head.append_u16(self.file_version)
head.append_u16(self.file_version ^ 0xFFFF)
head.append_u32(int(unixtime()))
head.append_u32(self.manifest.mem_size)
manifest_end = len(manifest_bin) + head.offset + 4
if self.file_version > 1:
manifest_end += 16
head.append_u32(manifest_end)
if self.file_version > 1:
head.append_bytes(manifest_hash)
ifs_file.write(head.data)
ifs_file.write(manifest_bin)
ifs_file.write(data)
ifs_file.close()
def _repack_tree(self, progress=True, **kwargs):
folders = self.tree.all_folders
files = self.tree.all_files
# Can't pickle lmxl, so to dirty-hack land we go
kbin_backup = []
for folder in folders:
if isinstance(folder, MD5Folder):
kbin_backup.append(folder.info_kbin)
folder.info_kbin = None
needs_preload = (f for f in files
if f.needs_preload or not kwargs['use_cache'])
args = list(zip(needs_preload, itertools.cycle((kwargs, ))))
p = Pool()
for f in tqdm(p.imap_unordered(_load, args),
desc='Caching',
total=len(args),
disable=not progress):
if progress:
tqdm.write(f)
# restore stuff from before
for folder in folders:
if isinstance(folder, MD5Folder):
folder.info_kbin = kbin_backup.pop(0)
tqdm_progress = None
if progress:
tqdm_progress = tqdm(desc='Writing', total=len(files))
self.tree.repack(self.manifest.xml_doc, self.data_blob, tqdm_progress,
**kwargs)
return self.data_blob.getvalue()