def zipper(self, ):
stream = BinaryStream()
assign_versions(stream, get_versions(self.ovl))
self.write_archive(stream)
uncompressed_bytes = stream.getbuffer()
# compress data
# change to zipped format for saving of uncompressed or oodled ovls
if not self.ovl.user_version.use_zlib:
print("HACK: setting compression to zlib")
self.ovl.user_version.use_oodle = False
self.ovl.user_version.use_zlib = True
# pc/pz zlib 8340 00100000 10010100
# pc/pz uncompressed 8212 00100000 00010100
# pc/pz oodle 8724 00100010 00010100
# JWE zlib 24724 01100000 10010100
# JWE oodle (switch) 25108 01100010 00010100
# vs = (8340, 8212, 8724, 24724, 25108)
# for v in vs:
# print(v)
# print(bin(v))
# print()
if self.ovl.user_version.use_oodle:
assert self.compression_header.startswith(OODLE_MAGIC)
a, raw_algo = struct.unpack("BB", self.compression_header)
algo = OodleDecompressEnum(raw_algo)
print("Oodle compression", a, raw_algo, algo.name)
compressed = texconv.oodle_compressor.compress(bytes(uncompressed_bytes), algo.name)
elif self.ovl.user_version.use_zlib:
compressed = zlib.compress(uncompressed_bytes)
else:
compressed = uncompressed_bytes
return len(uncompressed_bytes), len(compressed), compressed
def __init__(self, ovl): self.name = "NONE" self.buffer_info = None self.mdl2s = [] self.ovl = ovl self.versions = get_versions(self.ovl) self.ms2_entry = None self.bone_info = []
def update_buffer_0_bytes(self):
# update self.bone_names_size
with BinaryStream() as temp_writer:
assign_versions(temp_writer, get_versions(self))
temp_writer.ms_2_version = self.general_info.ms_2_version
self.buffer_0.write(temp_writer)
self.buffer_0_bytes = temp_writer.getvalue()
self.bone_names_size = len(self.buffer_0_bytes)
def load(self, ms2_file_path):
logging.info(f"Injecting MS2")
versions = get_versions(self.ovl)
ms2_file = Ms2File()
ms2_file.load(ms2_file_path, read_bytes=True)
missing_materials = set()
for model_info, mdl2_name, mdl2_entry in zip(ms2_file.model_infos, ms2_file.mdl_2_names, self.sized_str_entry.children):
for material in model_info.model.materials:
fgm_name = f"{material.name.lower()}.fgm"
if ovl_versions.is_jwe(self.ovl) or ovl_versions.is_jwe2(self.ovl) and fgm_name == "airliftstraps.fgm":
# don't cry about this
continue
if fgm_name not in self.ovl._ss_dict:
missing_materials.add(fgm_name)
if len(mdl2_entry.model_data_frags) != len(model_info.model.meshes):
raise AttributeError(
f"{mdl2_entry.name} ({len(model_info.model.meshes)}) doesn't have the "
f"expected amount ({len(mdl2_entry.model_data_frags)}) of meshes!")
if missing_materials:
mats = '\n'.join(missing_materials)
msg = f"The following materials are used by {self.file_entry.name}, but are missing from the OVL:\n" \
f"{mats}\n" \
f"This will crash unless you are importing the materials from another OVL. Inject anyway?"
if not interaction.showdialog(msg, ask=True):
logging.info("Injection was canceled by the user")
return
for mdl2_entry, model_info in zip(self.sized_str_entry.children, ms2_file.model_infos):
logging.debug(f"Injecting {mdl2_entry.name} ")
materials, lods, objects, meshes, model_info_ptr = mdl2_entry.fragments
for frag, mdl2_list in (
(materials, model_info.model.materials,),
(lods, model_info.model.lods),
(objects, model_info.model.objects),
(meshes, model_info.model.meshes)):
if len(mdl2_list) > 0:
data = as_bytes(mdl2_list, version_info=versions)
# objects.pointers[1] has padding in stock, apparently as each entry is 4 bytes
logging.debug(f"Injecting mdl2 data {len(data)} into {len(frag.pointers[1].data)} ({len(frag.pointers[1].padding)})")
# frag.pointers[1].update_data(data, pad_to=8)
# the above breaks injecting minmi
frag.pointers[1].update_data(data)
logging.debug(f"Result {len(frag.pointers[1].data)} ({len(frag.pointers[1].padding)})")
# load ms2 ss data
self.sized_str_entry.pointers[0].update_data(as_bytes(ms2_file.info, version_info=versions))
buffer_info_frag, model_info_frag, end_frag = self.sized_str_entry.fragments
buffer_info_frag.pointers[1].update_data(as_bytes(ms2_file.buffer_info, version_info=versions), update_copies=True)
model_info_frag.pointers[1].update_data(as_bytes(ms2_file.model_infos, version_info=versions))
# update ms2 data
self.sized_str_entry.data_entry.update_data(ms2_file.buffers)
def __init__(self, ovl): self.name = "NONE" self.lodinfo = b"" self.model_data_frags = [] self.ovl = ovl self.versions = get_versions(self.ovl) self.source = "NONE" self.mdl2_entry = None self.bone_info_buffer = None self.models = [] self.lods = []
def _get_frag_datas(self, fgm_data): versions = get_versions(self.ovl) sizedstr_bytes = as_bytes(fgm_data.fgm_info, version_info=versions) textures_bytes = as_bytes(fgm_data.textures, version_info=versions) attributes_bytes = as_bytes(fgm_data.attributes, version_info=versions) # todo - this is definitely NOT right/ needed padding by comparing to stock FGMs # no clue what the 'rule' here is, it may not be padding but be appear if another ptr is missing # textures_bytes += get_padding(len(textures_bytes), alignment=16) # attributes never seem to have padding # attributes_bytes += get_padding(len(attributes_bytes), alignment=16) fgm_header = fgm_data.fgm_info datas = [] if fgm_header.texture_count: datas.append(textures_bytes) if fgm_header.attribute_count: datas.append(attributes_bytes) datas.append(fgm_data.data_bytes) return datas, sizedstr_bytes
def load_fgm(ovl, fgm_file_path, fgm_sized_str_entry):
versions = get_versions(ovl)
fgm_data = FgmFile()
fgm_data.load(fgm_file_path)
sizedstr_bytes = as_bytes(
fgm_data.fgm_info, version_info=versions) + as_bytes(
fgm_data.two_frags_pad, version_info=versions)
# todo - move texpad into fragment padding?
textures_bytes = as_bytes(fgm_data.textures,
version_info=versions) + as_bytes(
fgm_data.texpad, version_info=versions)
attributes_bytes = as_bytes(fgm_data.attributes, version_info=versions)
# the actual injection
fgm_sized_str_entry.data_entry.update_data((fgm_data.buffer_bytes, ))
fgm_sized_str_entry.pointers[0].update_data(sizedstr_bytes,
update_copies=True)
if len(fgm_sized_str_entry.fragments) == 4:
datas = (textures_bytes, attributes_bytes, fgm_data.zeros_bytes,
fgm_data.data_bytes)
# fgms without zeros
elif len(fgm_sized_str_entry.fragments) == 3:
datas = (textures_bytes, attributes_bytes, fgm_data.data_bytes)
# fgms for variants
elif len(fgm_sized_str_entry.fragments) == 2:
datas = (attributes_bytes, fgm_data.data_bytes)
else:
raise AttributeError("Unexpected fgm frag count")
# inject fragment datas
for frag, data in zip(fgm_sized_str_entry.fragments, datas):
frag.pointers[1].update_data(data, update_copies=True)
# update dependencies on ovl
fgm_file_entry = get_file_entry(ovl, fgm_sized_str_entry)
for dep_entry, tex_name in zip(fgm_file_entry.dependencies,
fgm_data.texture_names):
dep_entry.basename = tex_name
dep_entry.name = dep_entry.basename + dep_entry.ext.replace(":", ".")
dep_entry.file_hash = djb(tex_name.lower())
def collect(self, ovl, file_entry):
self.ovl = ovl
ms2_entry = self.ovl.ss_dict[file_entry.name]
ss_pointer = ms2_entry.pointers[0]
self.ovs = ovl.static_archive.content
frags = self.ovs.pools[ss_pointer.pool_index].fragments
if not is_old(self.ovl):
ms2_entry.fragments = self.ovs.get_frags_after_count(
frags, ss_pointer.address, 3)
# print(ms2_entry.fragments)
# second pass: collect model fragments
versions = get_versions(self.ovl)
assert ms2_entry.pointers[0].data_size == 24
assert ms2_entry.fragments[2].pointers[1].data in (struct.pack(
"<ii", -1, 0), b"")
# print(ms2_entry.fragments[2].pointers[1].address, ms2_entry.fragments[2].pointers[1].data)
# assign the mdl2 frags to their sized str entry
# 3 fixed fragments laid out like
# sse p0: ms2_general_info_data (24 bytes)
# 0 - p0: 8*00 p1: buffer_info or empty (if no buffers)
# 1 - p0: 8*00 p1: core_model_info for first mdl2 file
# 2 - p0: 8*00 p1: 2 unk uints: -1, 0 or empty (if no buffers)
f_1 = ms2_entry.fragments[1]
core_model_info = f_1.pointers[1].load_as(CoreModelInfo,
version_info=versions)[0]
# print("next model info:", core_model_info)
for mdl2_entry in ms2_entry.children:
assert mdl2_entry.ext == ".mdl2"
self.collect_mdl2(mdl2_entry, core_model_info, f_1.pointers[1])
pink = mdl2_entry.fragments[4]
if (is_jwe(self.ovl) and pink.pointers[0].data_size == 144) \
or (is_pz(self.ovl) and pink.pointers[0].data_size == 160):
core_model_info = pink.pointers[0].load_as(
Mdl2ModelInfo, version_info=versions)[0].info
else:
ms2_entry.fragments = self.ovs.get_frags_after_count(
frags, ss_pointer.address, 1)
def get_tex_structs(self, sized_str_entry):
versions = get_versions(self.ovl)
tex_header = sized_str_entry.pointers[0].load_as(
TexHeader, version_info=versions)[0]
if is_pc(self.ovl) or is_ztuac(self.ovl):
frag = sized_str_entry.fragments[0]
tex_buffers = frag.pointers[1].load_as(TexBufferPc,
num=tex_header.stream_count,
version_info=versions)
# this corresponds to a stripped down header_7
header_7 = tex_buffers[0]
else:
# we have exactly two fragments, pointing into these pool_groups
f_3_3, f_3_7 = sized_str_entry.fragments
tex_buffers = f_3_3.pointers[1].load_as(
TexBuffer, num=tex_header.stream_count, version_info=versions)
header_7 = f_3_7.pointers[1].load_as(Header7Data1,
version_info=versions)[0]
# print(header_7)
return tex_header, tex_buffers, header_7
def create(self, ovs, file_entry):
self.ovs = ovs
self.ovl = ovs.ovl
ms2_file = Ms2File()
ms2_file.load(file_entry.path, read_bytes=True)
ms2_entry = self.create_ss_entry(file_entry)
ms2_entry.children = []
versions = get_versions(ovs.ovl)
pool_index, pool = self.get_pool(2)
offset = pool.data.tell()
ms2_dir, ms2_basename = os.path.split(file_entry.path)
mdl2_names = [
f for f in os.listdir(ms2_dir) if f.lower().endswith(".mdl2")
]
mdl2s = []
for mdl2_name in mdl2_names:
mdl2_path = os.path.join(ms2_dir, mdl2_name)
mdl2 = Mdl2File()
mdl2.load(mdl2_path)
if mdl2.ms_2_name == ms2_basename:
mdl2s.append((mdl2_name, mdl2))
# sort them by model index
mdl2s.sort(key=lambda tup: tup[1].index)
# create sized str entries and model data fragments
for mdl2_name, mdl2 in mdl2s:
mdl2_file_entry = self.get_file_entry(mdl2_name)
mdl2_entry = self.create_ss_entry(mdl2_file_entry)
mdl2_entry.pointers[0].pool_index = -1
mdl2_entry.pointers[0].data_offset = 0
ms2_entry.children.append(mdl2_entry)
# first, create all ModelData structs as fragments
mdl2_entry.model_data_frags = [
self.create_fragment()
for _ in range(mdl2.model_info.num_models)
]
# create the 5 fixed frags per MDL2 and write their data
for (mdl2_name, mdl2), mdl2_entry in zip(mdl2s, ms2_entry.children):
mdl2_entry.fragments = [self.create_fragment() for _ in range(5)]
materials, lods, objects, model_data_ptr, model_info = mdl2_entry.fragments
materials_offset = pool.data.tell()
materials.pointers[1].pool_index = pool_index
materials.pointers[1].data_offset = materials_offset
pool.data.write(as_bytes(mdl2.materials, version_info=versions))
lods.pointers[1].pool_index = pool_index
lods.pointers[1].data_offset = pool.data.tell()
pool.data.write(as_bytes(mdl2.lods, version_info=versions))
objects.pointers[1].pool_index = pool_index
objects.pointers[1].data_offset = pool.data.tell()
objects_bytes = as_bytes(mdl2.objects, version_info=versions)
pool.data.write(objects_bytes +
get_padding(len(objects_bytes), alignment=8))
# modeldatas start here
model_info.pointers[1].pool_index = pool_index
model_info.pointers[1].data_offset = materials_offset
# write modeldata
for (mdl2_name, mdl2), mdl2_entry in zip(mdl2s, ms2_entry.children):
materials, lods, objects, model_data_ptr, model_info = mdl2_entry.fragments
model_data_ptr.pointers[1].pool_index = pool_index
model_data_ptr.pointers[1].data_offset = pool.data.tell()
# write mdl2 modeldata frags
for frag, modeldata in zip(mdl2_entry.model_data_frags,
mdl2.models):
frag.pointers[0].pool_index = pool_index
frag.pointers[0].data_offset = pool.data.tell()
pool.data.write(as_bytes(modeldata, version_info=versions))
# create fragments for ms2
ms2_entry.fragments = [self.create_fragment() for _ in range(3)]
# write model info
for mdl2_name, mdl2 in mdl2s:
model_info_bytes = as_bytes(mdl2.model_info, version_info=versions)
if mdl2.index == 0:
f_0, f_1, f_2 = ms2_entry.fragments
f_1.pointers[1].pool_index = pool_index
f_1.pointers[1].data_offset = pool.data.tell()
# only write core model info
pool.data.write(model_info_bytes)
else:
# grab the preceding mdl2 entry since it points ahead
prev_mdl2_entry = ms2_entry.children[mdl2.index - 1]
# get its model info fragment
materials, lods, objects, model_data_ptr, model_info = prev_mdl2_entry.fragments
model_info.pointers[1].pool_index = pool_index
model_info.pointers[1].data_offset = pool.data.tell()
# we write this anyway
# todo - get the actual data
pool.data.write(b"\x00" * 40)
# we should only
pool.data.write(model_info_bytes)
this_mdl2_entry = ms2_entry.children[mdl2.index]
materials, lods, objects, model_data_ptr, model_info = this_mdl2_entry.fragments
for frag in (materials, lods, objects, model_data_ptr):
frag.pointers[0].pool_index = pool_index
frag.pointers[0].data_offset = pool.data.tell()
pool.data.write(b"\x00" * 8)
# write last 40 bytes to model_info
if mdl2s:
model_info.pointers[0].pool_index = pool_index
model_info.pointers[0].data_offset = pool.data.tell()
pool.data.write(b"\x00" * 40)
# write the ms2 itself
ms2_entry.pointers[0].pool_index = pool_index
ms2_entry.pointers[0].data_offset = pool.data.tell()
# load ms2 ss data
ms2_ss_bytes = as_bytes(
ms2_file.general_info,
version_info=versions) # + ms2_entry.pointers[0].data[24:]
pool.data.write(ms2_ss_bytes)
# first, 3 * 8 bytes of 00
for frag in ms2_entry.fragments:
frag.pointers[0].pool_index = pool_index
frag.pointers[1].pool_index = pool_index
frag.pointers[0].data_offset = pool.data.tell()
pool.data.write(b"\x00" * 8)
# now the actual data
buffer_info_frag, model_info_frag, end_frag = ms2_entry.fragments
buffer_info_offset = pool.data.tell()
# set ptr to buffer info for each ModelData frag
for mdl2_entry in ms2_entry.children:
for frag in mdl2_entry.model_data_frags:
frag.pointers[1].pool_index = pool_index
frag.pointers[1].data_offset = buffer_info_offset
# todo - from the frag log, buffer_info_bytes should be 48 bytes but is 32
buffer_info_frag.pointers[1].data_offset = buffer_info_offset
buffer_info_bytes = as_bytes(ms2_file.buffer_info,
version_info=versions)
logging.debug(f"len(buffer_info_bytes) {len(buffer_info_bytes)}")
pool.data.write(buffer_info_bytes)
# the last ms2 fragment
end_frag.pointers[1].data_offset = pool.data.tell()
pool.data.write(struct.pack("<ii", -1, 0))
# create ms2 data
self.create_data_entry(
file_entry, (ms2_file.buffer_0_bytes, ms2_file.buffer_1_bytes,
ms2_file.buffer_2_bytes))
def load_ms2(ovl_data, ms2_file_path, ms2_entry):
logging.info(f"Injecting MS2")
ms2_file = Ms2File()
ms2_file.load(ms2_file_path, read_bytes=True)
versions = get_versions(ovl_data)
# load ms2 ss data
ms2_ss_bytes = as_bytes(
ms2_file.general_info,
version_info=versions) + ms2_entry.pointers[0].data[24:]
ms2_entry.pointers[0].update_data(ms2_ss_bytes, update_copies=True)
# overwrite ms2 buffer info frag
buffer_info_frag = ms2_entry.fragments[0]
buffer_info_frag.pointers[1].update_data(as_bytes(ms2_file.buffer_info,
version_info=versions),
update_copies=True)
# update ms2 data
ms2_entry.data_entry.update_data([
ms2_file.buffer_0_bytes, ms2_file.buffer_1_bytes,
ms2_file.buffer_2_bytes
])
logging.info(f"Injecting MDL2s")
ms2_dir = os.path.dirname(ms2_file_path)
mdl2s = []
for mdl2_entry in ms2_entry.children:
mdl2_path = os.path.join(ms2_dir, mdl2_entry.name)
mdl2 = Mdl2File()
mdl2.load(mdl2_path)
mdl2s.append(mdl2)
if len(mdl2_entry.model_data_frags) != len(mdl2.models):
raise AttributeError(
f"{mdl2_entry.name} doesn't have the right amount of meshes!")
# overwrite mdl2 modeldata frags
for frag, modeldata in zip(mdl2_entry.model_data_frags, mdl2.models):
frag_data = as_bytes(modeldata, version_info=versions)
frag.pointers[0].update_data(frag_data, update_copies=True)
materials, lods, objects, model_data_ptr, model_info = mdl2_entry.fragments
for frag, mdl2_list in ((
materials,
mdl2.materials,
), (lods, mdl2.lods), (objects, mdl2.objects)):
if len(mdl2_list) > 0:
data = as_bytes(mdl2_list, version_info=versions)
frag.pointers[1].update_data(data,
update_copies=True,
pad_to=8)
for mdl2 in mdl2s:
data = as_bytes(mdl2.model_info, version_info=versions)
if mdl2.index == 0:
f_0, f_1, f_2 = ms2_entry.fragments
f_1.pointers[1].update_data(data, update_copies=True)
else:
# grab the preceding mdl2 entry since it points ahead
mdl2_entry = ms2_entry.children[mdl2.index - 1]
# get its model info fragment
materials, lods, objects, model_data_ptr, model_info = mdl2_entry.fragments
if (is_jwe(ovl_data) and model_info.pointers[0].data_size == 144) \
or (is_pz(ovl_data) and model_info.pointers[0].data_size == 160):
data = model_info.pointers[0].data[:40] + data
model_info.pointers[0].update_data(data, update_copies=True)
def create(self):
ms2_file = Ms2File()
ms2_file.load(self.file_entry.path, read_bytes=True)
ms2_dir = os.path.dirname(self.file_entry.path)
ms2_entry = self.create_ss_entry(self.file_entry)
ms2_entry.children = []
versions = get_versions(self.ovl)
# 1 for the ms2, 2 for each mdl2
# pool.num_files += 1
# create sized str entries and mesh data fragments
for model_info, mdl2_name in zip(ms2_file.model_infos, ms2_file.mdl_2_names):
# pool.num_files += 2
mdl2_path = os.path.join(ms2_dir, mdl2_name+".mdl2")
mdl2_file_entry = self.get_file_entry(mdl2_path)
mdl2_entry = self.create_ss_entry(mdl2_file_entry)
mdl2_entry.pointers[0].pool_index = -1
ms2_entry.children.append(mdl2_entry)
# first, create all MeshData structs as fragments
mdl2_entry.model_data_frags = [self.create_fragment() for _ in range(model_info.num_meshes)]
first_materials_ptr = None
# create the 5 fixed frags per MDL2 and write their data
for model_info, mdl2_entry in zip(ms2_file.model_infos, ms2_entry.children):
mdl2_entry.fragments = [self.create_fragment() for _ in range(5)]
materials, lods, objects, meshes, model_info_ptr = mdl2_entry.fragments
if first_materials_ptr is None:
first_materials_ptr = materials.pointers[1]
self.write_to_pool(materials.pointers[1], 2, as_bytes(model_info.model.materials, version_info=versions))
self.write_to_pool(lods.pointers[1], 2, as_bytes(model_info.model.lods, version_info=versions))
objects_bytes = as_bytes(model_info.model.objects, version_info=versions)
# todo - padding like this is likely wrong, probably relative to start of materials
self.write_to_pool(objects.pointers[1], 2, objects_bytes + get_padding(len(objects_bytes), alignment=8))
self.write_to_pool(meshes.pointers[1], 2, as_bytes(model_info.model.meshes, version_info=versions))
self.ptr_relative(model_info_ptr.pointers[1], first_materials_ptr)
# point to start of each modeldata
offset = 0
for frag in mdl2_entry.model_data_frags:
self.ptr_relative(frag.pointers[0], meshes.pointers[1], rel_offset=offset)
offset += 64
# create fragments for ms2
buffer_info_frag, model_info_frag, end_frag = self.create_fragments(ms2_entry, 3)
# write mesh info
self.write_to_pool(model_info_frag.pointers[1], 2, as_bytes(ms2_file.model_infos, version_info=versions))
offset = 0
for mdl2_entry in ms2_entry.children:
# byte size of modelinfo varies - JWE1 (176 bytes total)
if ovl_versions.is_jwe(self.ovl):
offset += 104
# 16 additional bytes for PZ/PZ16/JWE2 (192 bytes total)
else:
offset += 120
for frag in mdl2_entry.fragments:
self.ptr_relative(frag.pointers[0], model_info_frag.pointers[1], rel_offset=offset)
offset += 8
offset += 32
# buffer info data
buffer_info_bytes = as_bytes(ms2_file.buffer_info, version_info=versions)
self.write_to_pool(buffer_info_frag.pointers[1], 2, buffer_info_bytes)
# set ptr to buffer info for each MeshData frag
for mdl2_entry in ms2_entry.children:
for frag in mdl2_entry.model_data_frags:
self.ptr_relative(frag.pointers[1], buffer_info_frag.pointers[1])
# ms2 ss data
ms2_ss_bytes = as_bytes(ms2_file.info, version_info=versions)
self.write_to_pool(ms2_entry.pointers[0], 2, ms2_ss_bytes)
# set frag ptr 0
for frag, offset in zip(ms2_entry.fragments, (24, 32, 40)):
self.ptr_relative(frag.pointers[0], ms2_entry.pointers[0], rel_offset=offset)
# the last ms2 fragment
self.write_to_pool(end_frag.pointers[1], 2, struct.pack("<ii", -1, 0))
# create ms2 data
self.create_data_entry(ms2_entry, ms2_file.buffers)
def update_buffer_1_bytes(self):
with BinaryStream() as temp_bone_writer:
assign_versions(temp_bone_writer, get_versions(self))
temp_bone_writer.ms_2_version = self.general_info.ms_2_version
self.write_all_bone_infos(temp_bone_writer)
self.buffer_1_bytes = temp_bone_writer.getvalue()
