def zipper(self, ): stream = BinaryStream() assign_versions(stream, get_versions(self.ovl)) self.write_archive(stream) uncompressed_bytes = stream.getbuffer() # compress data # change to zipped format for saving of uncompressed or oodled ovls if not self.ovl.user_version.use_zlib: print("HACK: setting compression to zlib") self.ovl.user_version.use_oodle = False self.ovl.user_version.use_zlib = True # pc/pz zlib 8340 00100000 10010100 # pc/pz uncompressed 8212 00100000 00010100 # pc/pz oodle 8724 00100010 00010100 # JWE zlib 24724 01100000 10010100 # JWE oodle (switch) 25108 01100010 00010100 # vs = (8340, 8212, 8724, 24724, 25108) # for v in vs: # print(v) # print(bin(v)) # print() if self.ovl.user_version.use_oodle: assert self.compression_header.startswith(OODLE_MAGIC) a, raw_algo = struct.unpack("BB", self.compression_header) algo = OodleDecompressEnum(raw_algo) print("Oodle compression", a, raw_algo, algo.name) compressed = texconv.oodle_compressor.compress(bytes(uncompressed_bytes), algo.name) elif self.ovl.user_version.use_zlib: compressed = zlib.compress(uncompressed_bytes) else: compressed = uncompressed_bytes return len(uncompressed_bytes), len(compressed), compressed
def __init__(self, ovl): self.name = "NONE" self.buffer_info = None self.mdl2s = [] self.ovl = ovl self.versions = get_versions(self.ovl) self.ms2_entry = None self.bone_info = []
def update_buffer_0_bytes(self): # update self.bone_names_size with BinaryStream() as temp_writer: assign_versions(temp_writer, get_versions(self)) temp_writer.ms_2_version = self.general_info.ms_2_version self.buffer_0.write(temp_writer) self.buffer_0_bytes = temp_writer.getvalue() self.bone_names_size = len(self.buffer_0_bytes)
def load(self, ms2_file_path): logging.info(f"Injecting MS2") versions = get_versions(self.ovl) ms2_file = Ms2File() ms2_file.load(ms2_file_path, read_bytes=True) missing_materials = set() for model_info, mdl2_name, mdl2_entry in zip(ms2_file.model_infos, ms2_file.mdl_2_names, self.sized_str_entry.children): for material in model_info.model.materials: fgm_name = f"{material.name.lower()}.fgm" if ovl_versions.is_jwe(self.ovl) or ovl_versions.is_jwe2(self.ovl) and fgm_name == "airliftstraps.fgm": # don't cry about this continue if fgm_name not in self.ovl._ss_dict: missing_materials.add(fgm_name) if len(mdl2_entry.model_data_frags) != len(model_info.model.meshes): raise AttributeError( f"{mdl2_entry.name} ({len(model_info.model.meshes)}) doesn't have the " f"expected amount ({len(mdl2_entry.model_data_frags)}) of meshes!") if missing_materials: mats = '\n'.join(missing_materials) msg = f"The following materials are used by {self.file_entry.name}, but are missing from the OVL:\n" \ f"{mats}\n" \ f"This will crash unless you are importing the materials from another OVL. Inject anyway?" if not interaction.showdialog(msg, ask=True): logging.info("Injection was canceled by the user") return for mdl2_entry, model_info in zip(self.sized_str_entry.children, ms2_file.model_infos): logging.debug(f"Injecting {mdl2_entry.name} ") materials, lods, objects, meshes, model_info_ptr = mdl2_entry.fragments for frag, mdl2_list in ( (materials, model_info.model.materials,), (lods, model_info.model.lods), (objects, model_info.model.objects), (meshes, model_info.model.meshes)): if len(mdl2_list) > 0: data = as_bytes(mdl2_list, version_info=versions) # objects.pointers[1] has padding in stock, apparently as each entry is 4 bytes logging.debug(f"Injecting mdl2 data {len(data)} into {len(frag.pointers[1].data)} ({len(frag.pointers[1].padding)})") # frag.pointers[1].update_data(data, pad_to=8) # the above breaks injecting minmi frag.pointers[1].update_data(data) logging.debug(f"Result {len(frag.pointers[1].data)} ({len(frag.pointers[1].padding)})") # load ms2 ss data self.sized_str_entry.pointers[0].update_data(as_bytes(ms2_file.info, version_info=versions)) buffer_info_frag, model_info_frag, end_frag = self.sized_str_entry.fragments buffer_info_frag.pointers[1].update_data(as_bytes(ms2_file.buffer_info, version_info=versions), update_copies=True) model_info_frag.pointers[1].update_data(as_bytes(ms2_file.model_infos, version_info=versions)) # update ms2 data self.sized_str_entry.data_entry.update_data(ms2_file.buffers)
def __init__(self, ovl): self.name = "NONE" self.lodinfo = b"" self.model_data_frags = [] self.ovl = ovl self.versions = get_versions(self.ovl) self.source = "NONE" self.mdl2_entry = None self.bone_info_buffer = None self.models = [] self.lods = []
def _get_frag_datas(self, fgm_data): versions = get_versions(self.ovl) sizedstr_bytes = as_bytes(fgm_data.fgm_info, version_info=versions) textures_bytes = as_bytes(fgm_data.textures, version_info=versions) attributes_bytes = as_bytes(fgm_data.attributes, version_info=versions) # todo - this is definitely NOT right/ needed padding by comparing to stock FGMs # no clue what the 'rule' here is, it may not be padding but be appear if another ptr is missing # textures_bytes += get_padding(len(textures_bytes), alignment=16) # attributes never seem to have padding # attributes_bytes += get_padding(len(attributes_bytes), alignment=16) fgm_header = fgm_data.fgm_info datas = [] if fgm_header.texture_count: datas.append(textures_bytes) if fgm_header.attribute_count: datas.append(attributes_bytes) datas.append(fgm_data.data_bytes) return datas, sizedstr_bytes
def load_fgm(ovl, fgm_file_path, fgm_sized_str_entry): versions = get_versions(ovl) fgm_data = FgmFile() fgm_data.load(fgm_file_path) sizedstr_bytes = as_bytes( fgm_data.fgm_info, version_info=versions) + as_bytes( fgm_data.two_frags_pad, version_info=versions) # todo - move texpad into fragment padding? textures_bytes = as_bytes(fgm_data.textures, version_info=versions) + as_bytes( fgm_data.texpad, version_info=versions) attributes_bytes = as_bytes(fgm_data.attributes, version_info=versions) # the actual injection fgm_sized_str_entry.data_entry.update_data((fgm_data.buffer_bytes, )) fgm_sized_str_entry.pointers[0].update_data(sizedstr_bytes, update_copies=True) if len(fgm_sized_str_entry.fragments) == 4: datas = (textures_bytes, attributes_bytes, fgm_data.zeros_bytes, fgm_data.data_bytes) # fgms without zeros elif len(fgm_sized_str_entry.fragments) == 3: datas = (textures_bytes, attributes_bytes, fgm_data.data_bytes) # fgms for variants elif len(fgm_sized_str_entry.fragments) == 2: datas = (attributes_bytes, fgm_data.data_bytes) else: raise AttributeError("Unexpected fgm frag count") # inject fragment datas for frag, data in zip(fgm_sized_str_entry.fragments, datas): frag.pointers[1].update_data(data, update_copies=True) # update dependencies on ovl fgm_file_entry = get_file_entry(ovl, fgm_sized_str_entry) for dep_entry, tex_name in zip(fgm_file_entry.dependencies, fgm_data.texture_names): dep_entry.basename = tex_name dep_entry.name = dep_entry.basename + dep_entry.ext.replace(":", ".") dep_entry.file_hash = djb(tex_name.lower())
def collect(self, ovl, file_entry): self.ovl = ovl ms2_entry = self.ovl.ss_dict[file_entry.name] ss_pointer = ms2_entry.pointers[0] self.ovs = ovl.static_archive.content frags = self.ovs.pools[ss_pointer.pool_index].fragments if not is_old(self.ovl): ms2_entry.fragments = self.ovs.get_frags_after_count( frags, ss_pointer.address, 3) # print(ms2_entry.fragments) # second pass: collect model fragments versions = get_versions(self.ovl) assert ms2_entry.pointers[0].data_size == 24 assert ms2_entry.fragments[2].pointers[1].data in (struct.pack( "<ii", -1, 0), b"") # print(ms2_entry.fragments[2].pointers[1].address, ms2_entry.fragments[2].pointers[1].data) # assign the mdl2 frags to their sized str entry # 3 fixed fragments laid out like # sse p0: ms2_general_info_data (24 bytes) # 0 - p0: 8*00 p1: buffer_info or empty (if no buffers) # 1 - p0: 8*00 p1: core_model_info for first mdl2 file # 2 - p0: 8*00 p1: 2 unk uints: -1, 0 or empty (if no buffers) f_1 = ms2_entry.fragments[1] core_model_info = f_1.pointers[1].load_as(CoreModelInfo, version_info=versions)[0] # print("next model info:", core_model_info) for mdl2_entry in ms2_entry.children: assert mdl2_entry.ext == ".mdl2" self.collect_mdl2(mdl2_entry, core_model_info, f_1.pointers[1]) pink = mdl2_entry.fragments[4] if (is_jwe(self.ovl) and pink.pointers[0].data_size == 144) \ or (is_pz(self.ovl) and pink.pointers[0].data_size == 160): core_model_info = pink.pointers[0].load_as( Mdl2ModelInfo, version_info=versions)[0].info else: ms2_entry.fragments = self.ovs.get_frags_after_count( frags, ss_pointer.address, 1)
def get_tex_structs(self, sized_str_entry): versions = get_versions(self.ovl) tex_header = sized_str_entry.pointers[0].load_as( TexHeader, version_info=versions)[0] if is_pc(self.ovl) or is_ztuac(self.ovl): frag = sized_str_entry.fragments[0] tex_buffers = frag.pointers[1].load_as(TexBufferPc, num=tex_header.stream_count, version_info=versions) # this corresponds to a stripped down header_7 header_7 = tex_buffers[0] else: # we have exactly two fragments, pointing into these pool_groups f_3_3, f_3_7 = sized_str_entry.fragments tex_buffers = f_3_3.pointers[1].load_as( TexBuffer, num=tex_header.stream_count, version_info=versions) header_7 = f_3_7.pointers[1].load_as(Header7Data1, version_info=versions)[0] # print(header_7) return tex_header, tex_buffers, header_7
def create(self, ovs, file_entry): self.ovs = ovs self.ovl = ovs.ovl ms2_file = Ms2File() ms2_file.load(file_entry.path, read_bytes=True) ms2_entry = self.create_ss_entry(file_entry) ms2_entry.children = [] versions = get_versions(ovs.ovl) pool_index, pool = self.get_pool(2) offset = pool.data.tell() ms2_dir, ms2_basename = os.path.split(file_entry.path) mdl2_names = [ f for f in os.listdir(ms2_dir) if f.lower().endswith(".mdl2") ] mdl2s = [] for mdl2_name in mdl2_names: mdl2_path = os.path.join(ms2_dir, mdl2_name) mdl2 = Mdl2File() mdl2.load(mdl2_path) if mdl2.ms_2_name == ms2_basename: mdl2s.append((mdl2_name, mdl2)) # sort them by model index mdl2s.sort(key=lambda tup: tup[1].index) # create sized str entries and model data fragments for mdl2_name, mdl2 in mdl2s: mdl2_file_entry = self.get_file_entry(mdl2_name) mdl2_entry = self.create_ss_entry(mdl2_file_entry) mdl2_entry.pointers[0].pool_index = -1 mdl2_entry.pointers[0].data_offset = 0 ms2_entry.children.append(mdl2_entry) # first, create all ModelData structs as fragments mdl2_entry.model_data_frags = [ self.create_fragment() for _ in range(mdl2.model_info.num_models) ] # create the 5 fixed frags per MDL2 and write their data for (mdl2_name, mdl2), mdl2_entry in zip(mdl2s, ms2_entry.children): mdl2_entry.fragments = [self.create_fragment() for _ in range(5)] materials, lods, objects, model_data_ptr, model_info = mdl2_entry.fragments materials_offset = pool.data.tell() materials.pointers[1].pool_index = pool_index materials.pointers[1].data_offset = materials_offset pool.data.write(as_bytes(mdl2.materials, version_info=versions)) lods.pointers[1].pool_index = pool_index lods.pointers[1].data_offset = pool.data.tell() pool.data.write(as_bytes(mdl2.lods, version_info=versions)) objects.pointers[1].pool_index = pool_index objects.pointers[1].data_offset = pool.data.tell() objects_bytes = as_bytes(mdl2.objects, version_info=versions) pool.data.write(objects_bytes + get_padding(len(objects_bytes), alignment=8)) # modeldatas start here model_info.pointers[1].pool_index = pool_index model_info.pointers[1].data_offset = materials_offset # write modeldata for (mdl2_name, mdl2), mdl2_entry in zip(mdl2s, ms2_entry.children): materials, lods, objects, model_data_ptr, model_info = mdl2_entry.fragments model_data_ptr.pointers[1].pool_index = pool_index model_data_ptr.pointers[1].data_offset = pool.data.tell() # write mdl2 modeldata frags for frag, modeldata in zip(mdl2_entry.model_data_frags, mdl2.models): frag.pointers[0].pool_index = pool_index frag.pointers[0].data_offset = pool.data.tell() pool.data.write(as_bytes(modeldata, version_info=versions)) # create fragments for ms2 ms2_entry.fragments = [self.create_fragment() for _ in range(3)] # write model info for mdl2_name, mdl2 in mdl2s: model_info_bytes = as_bytes(mdl2.model_info, version_info=versions) if mdl2.index == 0: f_0, f_1, f_2 = ms2_entry.fragments f_1.pointers[1].pool_index = pool_index f_1.pointers[1].data_offset = pool.data.tell() # only write core model info pool.data.write(model_info_bytes) else: # grab the preceding mdl2 entry since it points ahead prev_mdl2_entry = ms2_entry.children[mdl2.index - 1] # get its model info fragment materials, lods, objects, model_data_ptr, model_info = prev_mdl2_entry.fragments model_info.pointers[1].pool_index = pool_index model_info.pointers[1].data_offset = pool.data.tell() # we write this anyway # todo - get the actual data pool.data.write(b"\x00" * 40) # we should only pool.data.write(model_info_bytes) this_mdl2_entry = ms2_entry.children[mdl2.index] materials, lods, objects, model_data_ptr, model_info = this_mdl2_entry.fragments for frag in (materials, lods, objects, model_data_ptr): frag.pointers[0].pool_index = pool_index frag.pointers[0].data_offset = pool.data.tell() pool.data.write(b"\x00" * 8) # write last 40 bytes to model_info if mdl2s: model_info.pointers[0].pool_index = pool_index model_info.pointers[0].data_offset = pool.data.tell() pool.data.write(b"\x00" * 40) # write the ms2 itself ms2_entry.pointers[0].pool_index = pool_index ms2_entry.pointers[0].data_offset = pool.data.tell() # load ms2 ss data ms2_ss_bytes = as_bytes( ms2_file.general_info, version_info=versions) # + ms2_entry.pointers[0].data[24:] pool.data.write(ms2_ss_bytes) # first, 3 * 8 bytes of 00 for frag in ms2_entry.fragments: frag.pointers[0].pool_index = pool_index frag.pointers[1].pool_index = pool_index frag.pointers[0].data_offset = pool.data.tell() pool.data.write(b"\x00" * 8) # now the actual data buffer_info_frag, model_info_frag, end_frag = ms2_entry.fragments buffer_info_offset = pool.data.tell() # set ptr to buffer info for each ModelData frag for mdl2_entry in ms2_entry.children: for frag in mdl2_entry.model_data_frags: frag.pointers[1].pool_index = pool_index frag.pointers[1].data_offset = buffer_info_offset # todo - from the frag log, buffer_info_bytes should be 48 bytes but is 32 buffer_info_frag.pointers[1].data_offset = buffer_info_offset buffer_info_bytes = as_bytes(ms2_file.buffer_info, version_info=versions) logging.debug(f"len(buffer_info_bytes) {len(buffer_info_bytes)}") pool.data.write(buffer_info_bytes) # the last ms2 fragment end_frag.pointers[1].data_offset = pool.data.tell() pool.data.write(struct.pack("<ii", -1, 0)) # create ms2 data self.create_data_entry( file_entry, (ms2_file.buffer_0_bytes, ms2_file.buffer_1_bytes, ms2_file.buffer_2_bytes))
def load_ms2(ovl_data, ms2_file_path, ms2_entry): logging.info(f"Injecting MS2") ms2_file = Ms2File() ms2_file.load(ms2_file_path, read_bytes=True) versions = get_versions(ovl_data) # load ms2 ss data ms2_ss_bytes = as_bytes( ms2_file.general_info, version_info=versions) + ms2_entry.pointers[0].data[24:] ms2_entry.pointers[0].update_data(ms2_ss_bytes, update_copies=True) # overwrite ms2 buffer info frag buffer_info_frag = ms2_entry.fragments[0] buffer_info_frag.pointers[1].update_data(as_bytes(ms2_file.buffer_info, version_info=versions), update_copies=True) # update ms2 data ms2_entry.data_entry.update_data([ ms2_file.buffer_0_bytes, ms2_file.buffer_1_bytes, ms2_file.buffer_2_bytes ]) logging.info(f"Injecting MDL2s") ms2_dir = os.path.dirname(ms2_file_path) mdl2s = [] for mdl2_entry in ms2_entry.children: mdl2_path = os.path.join(ms2_dir, mdl2_entry.name) mdl2 = Mdl2File() mdl2.load(mdl2_path) mdl2s.append(mdl2) if len(mdl2_entry.model_data_frags) != len(mdl2.models): raise AttributeError( f"{mdl2_entry.name} doesn't have the right amount of meshes!") # overwrite mdl2 modeldata frags for frag, modeldata in zip(mdl2_entry.model_data_frags, mdl2.models): frag_data = as_bytes(modeldata, version_info=versions) frag.pointers[0].update_data(frag_data, update_copies=True) materials, lods, objects, model_data_ptr, model_info = mdl2_entry.fragments for frag, mdl2_list in (( materials, mdl2.materials, ), (lods, mdl2.lods), (objects, mdl2.objects)): if len(mdl2_list) > 0: data = as_bytes(mdl2_list, version_info=versions) frag.pointers[1].update_data(data, update_copies=True, pad_to=8) for mdl2 in mdl2s: data = as_bytes(mdl2.model_info, version_info=versions) if mdl2.index == 0: f_0, f_1, f_2 = ms2_entry.fragments f_1.pointers[1].update_data(data, update_copies=True) else: # grab the preceding mdl2 entry since it points ahead mdl2_entry = ms2_entry.children[mdl2.index - 1] # get its model info fragment materials, lods, objects, model_data_ptr, model_info = mdl2_entry.fragments if (is_jwe(ovl_data) and model_info.pointers[0].data_size == 144) \ or (is_pz(ovl_data) and model_info.pointers[0].data_size == 160): data = model_info.pointers[0].data[:40] + data model_info.pointers[0].update_data(data, update_copies=True)
def create(self): ms2_file = Ms2File() ms2_file.load(self.file_entry.path, read_bytes=True) ms2_dir = os.path.dirname(self.file_entry.path) ms2_entry = self.create_ss_entry(self.file_entry) ms2_entry.children = [] versions = get_versions(self.ovl) # 1 for the ms2, 2 for each mdl2 # pool.num_files += 1 # create sized str entries and mesh data fragments for model_info, mdl2_name in zip(ms2_file.model_infos, ms2_file.mdl_2_names): # pool.num_files += 2 mdl2_path = os.path.join(ms2_dir, mdl2_name+".mdl2") mdl2_file_entry = self.get_file_entry(mdl2_path) mdl2_entry = self.create_ss_entry(mdl2_file_entry) mdl2_entry.pointers[0].pool_index = -1 ms2_entry.children.append(mdl2_entry) # first, create all MeshData structs as fragments mdl2_entry.model_data_frags = [self.create_fragment() for _ in range(model_info.num_meshes)] first_materials_ptr = None # create the 5 fixed frags per MDL2 and write their data for model_info, mdl2_entry in zip(ms2_file.model_infos, ms2_entry.children): mdl2_entry.fragments = [self.create_fragment() for _ in range(5)] materials, lods, objects, meshes, model_info_ptr = mdl2_entry.fragments if first_materials_ptr is None: first_materials_ptr = materials.pointers[1] self.write_to_pool(materials.pointers[1], 2, as_bytes(model_info.model.materials, version_info=versions)) self.write_to_pool(lods.pointers[1], 2, as_bytes(model_info.model.lods, version_info=versions)) objects_bytes = as_bytes(model_info.model.objects, version_info=versions) # todo - padding like this is likely wrong, probably relative to start of materials self.write_to_pool(objects.pointers[1], 2, objects_bytes + get_padding(len(objects_bytes), alignment=8)) self.write_to_pool(meshes.pointers[1], 2, as_bytes(model_info.model.meshes, version_info=versions)) self.ptr_relative(model_info_ptr.pointers[1], first_materials_ptr) # point to start of each modeldata offset = 0 for frag in mdl2_entry.model_data_frags: self.ptr_relative(frag.pointers[0], meshes.pointers[1], rel_offset=offset) offset += 64 # create fragments for ms2 buffer_info_frag, model_info_frag, end_frag = self.create_fragments(ms2_entry, 3) # write mesh info self.write_to_pool(model_info_frag.pointers[1], 2, as_bytes(ms2_file.model_infos, version_info=versions)) offset = 0 for mdl2_entry in ms2_entry.children: # byte size of modelinfo varies - JWE1 (176 bytes total) if ovl_versions.is_jwe(self.ovl): offset += 104 # 16 additional bytes for PZ/PZ16/JWE2 (192 bytes total) else: offset += 120 for frag in mdl2_entry.fragments: self.ptr_relative(frag.pointers[0], model_info_frag.pointers[1], rel_offset=offset) offset += 8 offset += 32 # buffer info data buffer_info_bytes = as_bytes(ms2_file.buffer_info, version_info=versions) self.write_to_pool(buffer_info_frag.pointers[1], 2, buffer_info_bytes) # set ptr to buffer info for each MeshData frag for mdl2_entry in ms2_entry.children: for frag in mdl2_entry.model_data_frags: self.ptr_relative(frag.pointers[1], buffer_info_frag.pointers[1]) # ms2 ss data ms2_ss_bytes = as_bytes(ms2_file.info, version_info=versions) self.write_to_pool(ms2_entry.pointers[0], 2, ms2_ss_bytes) # set frag ptr 0 for frag, offset in zip(ms2_entry.fragments, (24, 32, 40)): self.ptr_relative(frag.pointers[0], ms2_entry.pointers[0], rel_offset=offset) # the last ms2 fragment self.write_to_pool(end_frag.pointers[1], 2, struct.pack("<ii", -1, 0)) # create ms2 data self.create_data_entry(ms2_entry, ms2_file.buffers)
def update_buffer_1_bytes(self): with BinaryStream() as temp_bone_writer: assign_versions(temp_bone_writer, get_versions(self)) temp_bone_writer.ms_2_version = self.general_info.ms_2_version self.write_all_bone_infos(temp_bone_writer) self.buffer_1_bytes = temp_bone_writer.getvalue()