def parse_vb(self): def conv_tangent(v): return v / 255.0 * 2.0 - 1.0 vertices = [] vb = util.get_getter(self.vbufs[0], "<") stride = self.vb_strides[0] assert stride == 0x1C for i in xrange(self.vnum): # first vertex buffer x, y, z = vb.get("3f") tangent = map(conv_tangent, vb.get("4B")) # The last element being the `sign` for j in xrange(3): tangent[j] *= tangent[3] u, v = numpy.frombuffer(vb.get_raw(4), dtype=numpy.dtype("<f2")) bone_indices = vb.get("4B") bone_weights = vb.get("4B") #assert sum(bone_weights) == 0xFF, "bone_weights sum=0x%x" % sum(bone_weights) # secondary vertex buffer vb2 = util.get_getter(self.vbufs[1], "<") if self.vfmt2 == 10: # Offset Sematics Format # 0x0 TEXCOORD1 DXGI_FORMAT_R16G16_FLOAT # 0x8 NORMAL DXGI_FORMAT_R16G16B16A16_FLOAT vb2.skip(0x8) nx, ny, nz, nw = numpy.frombuffer(vb2.get_raw(8), dtype=numpy.dtype("<f2")) elif self.vfmt2 == 7: # Offset Sematics Format # 0x0 TEXCOORD1 DXGI_FORMAT_R16G16_FLOAT # 0x4 NORMAL DXGI_FORMAT_R16G16B16A16_FLOAT vb2.skip(0x4) nx, ny, nz, nw = numpy.frombuffer(vb2.get_raw(8), dtype=numpy.dtype("<f2")) elif self.vfmt2 == 11: # Offset Sematics Format # 0x0 TEXCOORD1 DXGI_FORMAT_R16G16_FLOAT # 0x8 NORMAL DXGI_FORMAT_R16G16B16A16_FLOAT # 0x10 TEXCOORD2 DXGI_FORMAT_R16G16_FLOAT vb2.skip(0x8) nx, ny, nz, nw = numpy.frombuffer(vb2.get_raw(8), dtype=numpy.dtype("<f2")) vb2.skip(0x4) vertices.append({ "position": (x, y, z), "normal": (nx, ny, nz), "uv": (u, v), "bone_indices": bone_indices, "bone_weights": bone_weights, }) return vertices
def get_g1mg_subchunk_data(data, schunk_type): for chunk_data in iter_chunk(data): get = get_getter(chunk_data, "<") chunk_name = get(0x0, "8s") chunk_size = get(0x8, "I") if chunk_name.startswith(G1MG): for subchunk_data in iter_g1mg_subchunk(chunk_data): sub_get = get_getter(subchunk_data, "<") if sub_get(0x0, "I") == schunk_type: return subchunk_data return None
def parse_vb(self): def f(v): return (v * 2 - 255.0) / 255.0 vertices = [] vb = util.get_getter(self.vbufs[0], "<") stride = self.vb_strides[0] assert stride == 0x1C for i in xrange(self.vnum): x, y, z = vb.get("3f") nx, ny, nz = map(f, vb.get("3B")) unknown = vb.get("B") u, v = numpy.frombuffer(vb.get_raw(4), dtype=numpy.dtype("<f2")) bone_indices = vb.get("4B") bone_weights = vb.get("4B") #assert sum(bone_weights) == 0xFF, "bone_weights sum=0x%x" % sum(bone_weights) vertices.append({ "position": (x, y, z), "normal": (nx, ny, nz), "unknown": unknown, "uv": (u, v), "bone_indices": bone_indices, "bone_weights": bone_weights, }) return vertices
def parse_primitives(mod, dp_info): print dp_info IA_d3d10 = IA_D3D10[str(dp_info.input_layout_index)] IA_game = IA_GAME[dp_info.input_layout_index] print "input_layout_index", dp_info.input_layout_index # Parse vertices = [] # parse referrenced vertex buffer vertex_format_size = calc_vertex_format_size(IA_d3d10) getter = util.get_getter(mod.vb, "<") getter.seek(dp_info.vb_offset + vertex_format_size * dp_info.index_min) for i in xrange(dp_info.index_min, dp_info.index_max + 1, 1): vertex = parse_vertex(getter, IA_d3d10, IA_game) if mod.bone_num > 0: unnormalize_vertex(vertex, mod.inv_norm_mat) vertices.append(vertex) used_x.add(vertex["Position"][0]) used_y.add(vertex["Position"][1]) used_z.add(vertex["Position"][2]) # for k, v in sorted(vertex.iteritems()): # print k, v # print "-" * 10 return vertices
def parse(spk_fpath): spk_f = open(spk_fpath, "rb") spk = util.get_getter(spk_f, "<") spk_header = spk.block(0x24) assert spk_header.get("4s") == "SPK\x00" assert spk_header.get("I") == 0xe11e1e22 assert spk_header.get("H") == 0x24 shader_pair_count = spk_header.get("H") vs_count = spk_header.get("H") ps_count = spk_header.get("H") gs_count = spk_header.get("H") input_layout_count = spk_header.get("H") assert vs_count == input_layout_count spk_header.skip(0x8) dxbc_size = spk_header.get("I") spk_info_size = spk_header.get("I") assert dxbc_size + spk_info_size == spk.size spk_header.assert_end() spk_metadata = spk.block(spk_info_size - 0x24) dxbc = spk.block(dxbc_size) spk.assert_end() spk_f.close() pair_info_list = get_shader_pair_info(spk_metadata, shader_pair_count) assert_shader_pair_info_list(pair_info_list, vs_count, ps_count, gs_count) dump_vertex_shaders(spk_metadata, dxbc, 0, vs_count) dump_pixel_shaders(spk_metadata, dxbc, 0, ps_count) dump_shader_pair_info(spk_metadata, shader_pair_count) return spk_header, spk_metadata, dxbc, pair_info_list
def iter_chunk(data): get = get_getter(data, "<") fourcc = get(0x0, "8s") assert fourcc == G1M_0036, "invalid g1m file!" filesize = get(0x8, "I") assert filesize == len(data), "file size not match, file may be corrupted!" headersize = get(0xc, "I") # usually the 1st chunk offset unk = get(0x10, "I") assert unk == 0x0, "this may not be a reserved field!" chunk_count = get(0x14, "I") # usually 5 chunks but not always off = headersize for i in xrange(chunk_count): chunk_name = get(off, "8s") chunk_size = get(off + 0x8, "I") print "chunk: %s, size: 0x%x" % (chunk_name, chunk_size) chunk_data = data[off:off + chunk_size] yield chunk_data off += chunk_size assert off == filesize, "invalid file size not match!!"
def parse(path): f = open(path, "rb") getter = util.get_getter(f, "<") # read header header = CMfxHeader() header.read(getter) # read string table getter.seek(header.string_table_offset) raw_string_table = getter.get_raw(getter.size - getter.offset) string_table = CStringTable(raw_string_table) # read MfxEntry mfx_entries = [] for i, mfx_entry_offset in enumerate(header.mfx_entry_offsets): getter.seek(mfx_entry_offset) mfx_entry = CMfxEntry() print i, mfx_entry.read(getter, string_table) mfx_entries.append(mfx_entry) f.close() fout = open("windbg/input_layouts2.json", "w") input_layouts = [{}] for mfx_entry in mfx_entries: input_layouts.append(mfx_entry.input_layout) while input_layouts and not input_layouts[-1]: input_layouts.pop(-1) json.dump(input_layouts, fout, indent=2) fout.close()
def tex_extract(path): f = open(path, "rb") data = f.read() f.close() get = util.get_getter(data, "<") fourcc = get(0x0, "8s") assert fourcc == G1TG0060, "invalid or unsupported g1t file!" file_size, header_size = get(0x8, "II") tex_count = get(0x10, "I") tex_off_list = get(header_size, "%dI" % tex_count, force_tuple=True) base_off = header_size # split for i in xrange(tex_count): out_path = path + ".tex%d" % i if not os.path.exists(out_path): off = base_off + tex_off_list[i] end_off = file_size if (i + 1 >= tex_count) else ( base_off + tex_off_list[i + 1]) tex_blk = data[off:end_off] fout = open(out_path, "wb") fout.write(tex_blk) fout.close() # convert for i in xrange(tex_count): out_path = path + ".tex%d" % i f = open(out_path, "rb") tex_blk = f.read() f.close() print "conv texture %d" % i conv_tex(tex_blk, out_path)
def parse_g1mg_subchunk_0x10005(schunk_data): fvf_list = [] get = get_getter(schunk_data, "<") schunk_type, schunk_size = get(0x0, "2I") fvf_count = get(0x8, "I") log("fvf count", fvf_count, lv=0) off = 0xc tot_vb_ref_list = [] for i in xrange(fvf_count): log("fvf %d" % i, lv=0) vb_ref_count = get(off + 0x0, "I") vb_ref_list = get(off + 0x4, "%dI" % vb_ref_count, force_tuple=True) tot_vb_ref_list.extend(vb_ref_list) attr_count = get(off + 0x4 + vb_ref_count * 0x4, "I") log("off = 0x%x, attr_count: %d, vb_ref_list:" % (off, attr_count), vb_ref_list, lv=0) off += 0x8 + vb_ref_count * 0x4 attrs = [] for j in xrange(attr_count): vb_ref_idx, offset = get(off + 0x0, "2H") assert 0 <= vb_ref_idx < vb_ref_count data_type = get(off + 0x4, "H") sematics = get(off + 0x6, "H") log("\tvb_ref_idx=%d, off=0x%x, datatype=0x%x__%s, sematics=0x%x__"\ "%s" % (vb_ref_idx, offset, data_type, DATA_TYPE_MAP.get(data_type, "_UNK_"), sematics, SEMATIC_NAME_MAP.get(sematics, "_UNK_")), lv=0) off += 0x8 attrs.append( (offset, sematics, data_type, vb_ref_idx, vb_ref_list)) fvf_list.append(attrs) assert tot_vb_ref_list == range(len(tot_vb_ref_list)) return {"fvf_list": fvf_list}
def do_test_isize(fpath): fp = open(fpath, "rb") wmb = util.get_getter(fp, "<") res = parse_isize(wmb) fp.close() if res: print fpath
def extract(path): f = open(path, "rb") data = f.read() f.close() get = get_getter(data, "<") fourcc = get(0x0, "4s") assert fourcc == EARC, "invalid EARC file!" unk = get(0x4, "I") if unk != 0x01: assert False, "warning: not 0x01" data_size = get(0x8, "I") header_size = get(0xc, "I") filelist_size = get(0x10, "I") file_count = get(0x14, "I") unk = get(0x18, "I") # assertion fails on PC version # if unk != 1: # assert False, "warning: not 1" fixed_offset = get(0x1c, "I") root = os.path.splitext(path)[0] if not os.path.exists(root): os.mkdir(root) base_offset = 0x1c last_off = None for i in xrange(file_count): off, size, filename = get(base_offset + i * 0x38, "2I48s") filename = filename.rstrip("\x00") # print "index: 0x%x raw_off: 0x%x" % (i, off) # we should always use calculated offset for extracting file # offset in the file will cause empty gap in memory, which is reserved for thoese 'dummy'(I guess) off = (off - fixed_offset + header_size + filelist_size) if last_off is not None and off != last_off: off = last_off end_off = off + size last_off = end_off # print "%03d: %s @0x%x -> 0x%x" % (i, filename, off, end_off) assert filename == "dummy" or size != 0, "empty file always use name 'dummy'" if size == 0: continue out_path = os.path.join(root, filename) fout = open(out_path, "wb") fout.write(data[off: off + size]) fout.close() assert end_off == len(data), "file size not invalid!! 0x%x vs 0x%x" % (end_off, len(data))
def _parse_chunck(in_path, prefix, parse_func, *args): fin = open(in_path, "rb") data = fin.read() fin.close() for chunk_data in iter_chunk(data): get = get_getter(chunk_data, "<") chunk_name = get(0x0, "8s") chunk_size = get(0x8, "I") if chunk_name.startswith(prefix): return parse_func(chunk_data, *args)
def handle_wmb(fpath, out_dir): fp = open(fpath, "rb") wmb = util.get_getter(fp, "<") wmb = wmb_parser.parse(wmb) fp.close() wmb_parser.dump_wmb( wmb, outpath=os.path.join(out_dir, os.path.split(fpath)[1].replace(".wmb", ".gtb")) )
def handle_dds(wtp_path, wta_path, out_dir): if wta_path: fp = open(wta_path, "rb") wta_reader = util.get_getter(fp, "<") wta = wta_parser.parse(wta_reader) fp.close() tex_hashes = wta.texture_hashes print tex_hashes else: tex_hashes = [] splitdds.split(wtp_path, tex_hashes, out_dir)
def parse_g1mm(data): get = get_getter(data, "<") fourcc = get(0x0, "8s") assert fourcc == G1MM0020, "invalid fourcc!!" chunk_size = get(0x8, "I") assert chunk_size == len(data), "ok" mat_count = get(0xc, "I") for i in xrange(mat_count): mat = get(0x10 + 0x40 * i, "16f") for j in xrange(4): print mat[j * 4:(j + 1) * 4]
def collect_version(path): version_dict = {} for version in (0x20160116, 0x20151123, 0x20151001, 0x10000): version_dict[version] = [] for fpath in util.iter_path(path): if fpath.endswith(".wmb"): print "processing:", fpath fp = open(fpath, "rb") wmb = util.get_getter(fp, "<") version_dict[wmb.get("I", offset=0x4)].append(os.path.split(fpath)[1]) fp.close() return version_dict
def convert(fpath): fp = open(fpath, "rb") bxm_reader = util.get_getter(fp, ">") node_infos, keyvals = parse(bxm_reader) fp.close() root = gen_xml(0, node_infos, keyvals) xml_str = etree.tostring(root, pretty_print=True, encoding="UTF-8") fp = open(fpath.replace(".bxm", ".xml"), "w") print fp fp.write(xml_str) fp.close()
def parse(data): get = get_getter(data, ">") off = 0x0 i = 0 while off < len(data): str_len = get(off, "b") if str_len == -1: break str_text = data[off + 1:off + 1 + str_len] off += 1 + str_len print "%3d:" % i, str_text i += 1
def parse_g1mg_subchunk_0x10001(schunk_data): dump_chunk = True get = get_getter(schunk_data, "<") schunk_type, schunk_size = get(0x0, "2I") entry_count = get(0x8, "I") reserved = get(0xc, "I") assert reserved == 0x100 assert len(schunk_data) == 0x10 + entry_count * 0x40 log("entry_count=%d" % entry_count, lv=0) # entry size == 0x40, but not matrix if dump_chunk: dump_data("g1mg_0x10001.bin", schunk_data)
def test(self): vb0 = util.get_getter(self.vbufs[0], "<") def f(v): return (v * 2 - 255.0) / 255.0 for i in xrange(self.vnum): vb0.seek(i * self.vb_strides[0]) vb0.skip(0xc) # position normal = map(f, vb0.get( "3B")) # mostly normalized, with some custom normals ... unk0 = vb0.get("B") vb0.skip(0xc) # uv, bone indices, bone weights assert unk0 in (0x0, 0xff)
def parse(path): load_input_layouts("windbg/input_layouts.json", "windbg/input_layouts2.json") f = open(path, "rb") getter = util.get_getter(f, "<") model = CModel() model.read(getter) f.close() model.dump(path.replace(".mod", ".gtb")) print_bounding_box_check(model)
def get_vertex_data_by_datatype(data, offset, datatype): get = get_getter(data, "<") if datatype == "float": return get(offset, "f") elif datatype == "float2": return get(offset, "2f") elif datatype == "float3": return get(offset, "3f") elif datatype == "float4": return get(offset, "4f") elif datatype == "int4": return get(offset, "4B") elif datatype == "RGBA": return hex(get(offset, "I")) assert False, "impossible"
def test_vdata_ex(path): f = open(path, "rb") data = f.read() f.close() get = get_getter(data, ">") vert_num = get(0xc, "I") stride = 0x8 offset_beg = get(0x1c, "I") offset_end = offset_beg + vert_num * stride for offset in xrange(offset_beg, offset_end, stride): c = get(offset, "I") u1, v1 = numpy.frombuffer(buffer(data[offset + 0x4: offset + 0x8]), dtype=numpy.dtype(">f2")) print "color:ARGB = 0x%08x, u1 = %.2f, v2 = %.2f" % (c, u1, v1)
def print_frame_info_0x4(data, offset, n): get = get_getter(data, ">") unk_header_size = 24 for i in xrange(3): unk_f0 = get(offset + i * 0x8, "f") print ("%.4f\t\t" % unk_f0), hex_format(data[offset + i * 0x8 + 0x4: offset + i * 0x8 + 0x8]) offset += unk_header_size for i in xrange(n): base_offset = offset + i * 0x8 unk_index, unk1, unk2, unk3 = get(base_offset, "4H") base_offset += 0x2 unk_floats = numpy.frombuffer(buffer(data[base_offset : base_offset + 0x6]), dtype=numpy.dtype(">f2")) print ("F 0x%02x:\t\t" % unk_index), hex(unk1), hex(unk2), hex(unk3) print
def parse_bone_names(path): f = open(path, "rb") data = f.read() f.close() get = get_getter(data, ">") names = [] off = 0 while off < len(data): str_len = get(off, "b") if str_len == -1: break names.append(data[off + 1:off + 1 + str_len]) off += 1 + str_len return names
def parse(lmt_path, out_path="objs/motion.gtba"): f = open(lmt_path, "rb") getter = util.get_getter(f, "<") lmt = LMT() lmt.read(getter) f.close() if EXPORT_SEPARATE_FILE: for motion_i in xrange(len(lmt.motion_list)): dump_single(lmt, motion_i, out_path.replace(".gtba", "_%d.gtba" % motion_i)) else: dump_all(lmt, out_path) return lmt
def iter_g1mg_subchunk(data): get = get_getter(data, "<") chunk_name = get(0x0, "8s") chunk_size = get(0x8, "I") platform = get(0xc, "4s") assert platform == "DX9\x00", "platform error!" unk = get(0x10, "I") assert unk == 0x0, "null padding value!" bbox = get(0x14, "6f") # (xmin, ymin, zmin, xmax, ymax, zmax) print "bounding box:", bbox schunk_count = get(0x2c, "I") off = 0x30 for i in xrange(schunk_count): schunk_type, schunk_size = get(off, "2I") yield data[off:off + schunk_size] off += schunk_size
def parse_g1mg_subchunk_0x10006(schunk_data): dump_chunk = False get = get_getter(schunk_data, "<") schunk_type, schunk_size = get(0x0, "2I") entry_count = get(0x8, "I") off = 0xc log("entry_count=%d" % entry_count, lv=0) for entry_idx in xrange(entry_count): item_count = get(off, "I") mat_ref_idx, unk0, unk1, unk2, joint_map_idx = get(off + 0x4, "IHHHH") assert unk0 == 0x8000 and unk2 == 0x8000 #count(locals(), "unk1") off += 0x4 + item_count * 0xc assert off == len(schunk_data) if dump_chunk: dump_data("g1mg_0x10006.bin", schunk_data)
def parse_g1mg_subchunk_0x10002(schunk_data): log("========", lv=1) log("materials", lv=1) log("========", lv=1) get = get_getter(schunk_data, "<") schunk_type, schunk_size = get(0x0, "2I") mat_count = get(0x8, "I") # dump_data("g1mg_0x10002.bin", schunk_data) off = 0xc material_list = [] for mat_idx in xrange(mat_count): unk0 = get(off + 0x0, "I") assert unk0 == 0 tex_count = get(off + 0x4, "I") unk1, unk2 = get(off + 0x8, "Ii") unk1_equal_tex_count = tex_count == unk1 count(locals(), "unk1_equal_tex_count") log("mat %d, tex_count %d, unk1=%d, unk2=%d, unk1_equal_tex_count=%d" % (mat_idx, tex_count, unk1, unk2, unk1_equal_tex_count), lv=1) assert 1 <= unk1 <= 7 assert unk2 == 1 or unk2 == -1 off += 0x10 material = {"texture_count": tex_count, "textures": []} material_list.append(material) for tex_idx in xrange(tex_count): tex_identifier = get(off + 0x0, "H") uv_chnl_idx, unk6 = get(off + 0x2, "HH") unk3, unk4, unk5 = get(off + 0x6, "3H") count(locals(), "unk6") assert 0 <= unk3 <= 2 assert unk4 == 4 assert unk5 == 4 assert 0 <= uv_chnl_idx <= 4, "works for this game!" off += 0xc log("tex_idx = %d, uv_channel_idx = %d, unk6 = %d, unk3 = %d, unk4 = %d, unk5 = %d" % (tex_identifier, uv_chnl_idx, unk6, unk3, unk4, unk5), lv=1) material["textures"].append([tex_identifier, uv_chnl_idx]) log("") return {"material_list": material_list}
def parse(f): data = f.read() get = get_getter(data, ">") FOURCC = get(0x0, "4s") assert FOURCC == "mot\x00", "invalid mot file" a, b, header_size, entry_count = get(0x4, "HHII") assert header_size == 0x10, "header_size != 0x10" print "%d, 0x%x, %d" % (a, b, get(0x4, "I")) entry_end = (header_size + entry_count * 0xc) print "entry_num = %d, entry_end = %d" % (entry_count, entry_end) base_offset = header_size last_bone_index = None last_frame_index = None now_off = entry_end offset_list = [] for i in xrange(entry_count): values = get(base_offset + i * 0xc, "h2b2h") bone_index = values[0] frame_index = values[1] if i == entry_count - 1: break if bone_index != last_bone_index: print else: assert frame_index >= last_frame_index int_impl = get(base_offset + i * 0xc + 0x8, "I") float_impl = get(base_offset + i * 0xc + 0x8, "f") if values[2] == 0: v = float_impl print_track_header(values[:5] + (v, )) else: v = int_impl offset_list.append(v) print_track_header(values[:5] + (v, )) #assert now_off == v, "expect off=%d, off=%d" % (now_off, v) f = 1 now_off += f * (12 + 4 * values[3]) if values[2] == 4: print_frame_info_0x4(data, offset_list[-1], values[3]) elif values[2] == 6: print_frame_info_0x6(data, offset_list[-1], values[3]) elif values[2] == 1: print_frame_info_0x1(data, offset_list[-1], values[3]) last_bone_index = bone_index last_frame_index = frame_index
def parse_g1mg_subchunk_0x10007(schunk_data): print "index buffer block" get = get_getter(schunk_data, "<") schunk_type, schunk_size = get(0x0, "2I") ib_count = get(0x8, "I") off = 0xc index_buffer_list = [] for j in xrange(ib_count): index_count, b, c = get(off, "3I") try: index_buffer_list.append(get(off + 0xc, "%dH" % index_count)) except struct.error, e: dump_data("g1mg_0x10007.bin", schunk_data) raise e off += 0xc + index_count * 2 print "%d => index_count: %d, 0x%x" % (j, index_count, b)
def parse_g1mg_subchunk_0x10004(schunk_data): get = get_getter(schunk_data, "<") schunk_type, schunk_size = get(0x0, "2I") mesh_seg_count = get(0x8, "I") off = 0xc vertex_buffer_list = [] for j in xrange(mesh_seg_count): unk1, fvf_size, vcount, unk2 = get(off, "4I") print "%d => mesh segment @ offset=0x%x, vcount=%d, fvf_size=0x%x, unk1=%d, unk2=%d" % ( j, off, vcount, fvf_size, unk1, unk2) assert unk1 == 0 and (unk2 == 0 or unk2 == 1), "unknown values!!" vertex_buffer = (fvf_size, vcount, unk2, off + 0x10, schunk_data) vertex_buffer_list.append(vertex_buffer) off += 0x10 + vcount * fvf_size return {"vertex_buffer_list": vertex_buffer_list}
def do_test(fpath): print "processing:", fpath fp = open(fpath, "rb") wmb = util.get_getter(fp, "<") wmb = parse(wmb) fp.close() if args.dry_run: return if args.format == "gtb": dump_wmb(wmb, outpath=fpath.replace(".wmb", ".gtb")) else: for lodlv in xrange(DUMP_MAX_LOD + 1): lod_info = wmb.lod_infos[lodlv] for submesh_idx in xrange(lod_info.submesh_start, lod_info.submesh_start + lod_info.submesh_num): outpath = lod_info.name + "_" + str(submesh_idx) + ".obj" dump_submesh(wmb.submesh_infos[submesh_idx], wmb.geo_buffers, outpath)
def parse_g1mg(data): g1mg = {} for schunk_data in iter_g1mg_subchunk(data): get = get_getter(schunk_data, "<") schunk_type, schunk_size = get(0x0, "2I") print "chunk_type=0x%x, chunk_size=0x%x" % (schunk_type, schunk_size) assert schunk_type in xrange( 0x10001, 0x1000A), "not recognized schunk type !! 0x%x" % schunk_type handler = G1MG_SUBCHUNK_HANDLER.get(schunk_type) if handler is None: continue ret = handler(schunk_data) if ret is not None: g1mg.update(ret) return g1mg
def extract_g1hp(g1hp_data): print "extracting" get = get_getter(g1hp_data, ">") fourcc = get(0x0, "8s") assert fourcc == "G1HP0010", "invalid fourcc" chunk_size = get(0x8, "I") assert len(g1hp_data) - chunk_size < 0x10, "invalid chunk size! 0x%x vs 0x%x" % (len(g1hp_data), chunk_size) unk0 = get(0xc, "I") g1m_count, unk1 = get(0x10, "2H") g1m_offset_list = get(0x14, "%dI" % g1m_count) print "offset_list", g1m_offset_list g1m_data = [] for i in xrange(g1m_count): off = g1m_offset_list[i] g1m_size = get(off + 0x8, "I") g1m_data.append(g1hp_data[off: off + g1m_size]) return g1m_data
def dump_obj(in_path, out_path): fin = open(in_path, "rb") data = fin.read() fin.close() obj_text = "" for chunk_data in iter_chunk(data): get = get_getter(chunk_data, "<") chunk_name = get(0x0, "8s") chunk_size = get(0x8, "I") if chunk_name.startswith(G1MG): g1mg = parse_g1mg(chunk_data) obj_text = g1m_export.export_obj(g1mg) break if obj_text: fout = open(out_path, "w") fout.write(obj_text) fout.close() return None
def extract(data, path): g1m_data = [] get = get_getter(data, ">") fourcc = get(0x0, "8s") assert fourcc == "G1H_0020", "invalid fourcc" file_size = get(0x8, "I") assert len(data) == file_size, "file size not match!" unk0, g1hp_chunk_count = get(0xc, "2H") assert unk0 == 0x10, "hey, guess wrong, this value has special meaning" g1hp_chunk_offset_list = get(0x10, "%dI" % g1hp_chunk_count) for i in xrange(g1hp_chunk_count): off = g1hp_chunk_offset_list[i] chunk_size = get(off + 0x8, "I") g1m_data = extract_g1hp(data[off: off + chunk_size]) for j in xrange(len(g1m_data)): dump_data(path.replace(".g1h", "_%d_%d.g1m" % (i, j)), g1m_data[j])
def parse_g1mg_subchunk_0x10003(schunk_data): log("================", lv=0) log("uniforms", lv=0) log("================", lv=0) dump_chunk = False get = get_getter(schunk_data, "<") schunk_type, schunk_size = get(0x0, "2I") uniform_blk_cnt = get(0x8, "I") offset = 0xc for uniform_blk_idx in xrange(uniform_blk_cnt): uniform_cnt = get(offset + 0x0, "I") log("\nuniform block %d: uniform_num=%d" % (uniform_blk_idx, uniform_cnt), lv=0) offset += 0x4 for uniform_idx in xrange(uniform_cnt): tot_len, name_len = get(offset, "2I") reserved0, datatype, reserved1 = get(offset + 0x8, "I2H") assert reserved0 == 0 and reserved1 == 1 name = get(offset + 0x10, "%ds" % name_len).rstrip("\x00") rem_size = tot_len - 0x10 - name_len if 1 <= datatype <= 4: vec_size = datatype assert rem_size == vec_size * 0x4 values = get(offset + 0x10 + name_len, "%df" % vec_size, force_tuple=True) values_string = ",".join(["%.4f" % v for v in values]) elif datatype == 5: assert rem_size == 4 values = get(offset + 0x10 + name_len, "4B", force_tuple=True) values_string = ",".join(["%d" % v for v in values]) else: assert False log("\tuniform: %s, values=%s, datatype=%d" % (name, values_string, datatype), lv=0) offset += tot_len if dump_chunk: dump_data("g1mg_0x10003.bin", schunk_data)
def unpack_file(fname): f = open(fname, "rb") data = f.read() f.close() get = get_getter(data, ">") FOURCC = get(0x0, "4s") assert FOURCC == "DAT\x00" file_count = get(0x4, "I") blk_file_off_off, blk_ext_off, blk_filename_off, blk_file_size_off = get(0x8, "IIII") dummys = get(0x18, "II") assert not any(dummys) # basicly no use ext_list = [] offset = blk_ext_off for i in xrange(file_count): term_off = data.find("\x00", offset) ext = data[offset: term_off] offset = term_off + 1 ext_list.append(ext) # create folders to put unpacked files in folder = os.path.splitext(fname)[0] if not os.path.isdir(folder): os.mkdir(folder) # extract files filename_len = get(blk_filename_off, "I") for i in xrange(file_count): name = get(blk_filename_off + 0x4 + i * filename_len, "%ds" % filename_len).rstrip("\x00") file_off = get(blk_file_off_off + i * 0x4, "I") file_size = get(blk_file_size_off + i * 0x4, "I") if file_size > 0: file_data = data[file_off: file_off + file_size] file_path = os.path.join(folder, name) fout = open(file_path, "wb") fout.write(file_data) fout.close()
def test_vf(path): f = open(path, "rb") data = f.read() f.close() get = get_getter(data, ">") vert_num = get(0xc, "I") stride = 0x20 offset_beg = get(0x18, "I") offset_end = offset_beg + vert_num * stride for offset in xrange(offset_beg, offset_end, stride): # pos x, y, z = get(offset, "fff") # uv u, v = numpy.frombuffer(buffer(data[offset + 0xc: offset + 0x10]), dtype=numpy.dtype(">f2")) # bone_indices bone_indices = get(offset + 0x18, "BBBB") # bone_weights bone_weights = get(offset + 0x1c, "BBBB") n1, n2, n3, n4 = get(offset + 0x10, "HHHH") u1, v1 = numpy.frombuffer(buffer(data[offset + 0x10: offset + 0x14]), dtype=numpy.dtype(">f2")) u2, v2 = numpy.frombuffer(buffer(data[offset + 0x14: offset + 0x18]), dtype=numpy.dtype(">f2")) nx, ny = get(offset + 0x10, "ff") #print "test 0x%04x, 0x%04x, 0x%04x, 0x%04x" % (n1, n2, n3, n4) # wrong #print "test2 %.2f, %.2f, %.2f, %.2f" % (u1, v1, u2, v2) #print "uv1", u1, v1 #print "uv2", u2, v2 # wrong #print "test3 %f, %f" % (nx, ny) print "0x%08x 0x%08x" % get(offset + 0x10, "II")
def parse(path): with open(path, "rb") as f: data = f.read() getter = util.get_getter(data, "<") header = getter.block(0x18) fourcc = header.get("4s") assert fourcc == "XFS\x00" unk0 = header.get("H") unk1 = header.get("H") unk_cnt = header.get("I") assert unk0 & 0x7FFF == 0xF subdata_cnt = header.get("I", offset=0x10) subdatablock_sz = header.get("I", offset=0x14) subdatablock = getter.block(subdatablock_sz) off_lst = subdatablock.get("%dI" % subdata_cnt, force_tuple=True) xml_defs = [] for i, off in enumerate(off_lst): if i + 1 < len(off_lst): sz = off_lst[i + 1] - off else: sz = subdatablock_sz - off subdatablock.seek(off) subdata = subdatablock.block(sz) def_ = parse_subdata(subdatablock, subdata) xml_defs.append(def_) n = getter.get("I") assert ((n >> 1) & 0x7FFF) != 0x7FFF assert n == 1 sz = getter.get("I") blk = getter.block(sz - 4) parse_object(xml_defs, 0, blk)
def unpack(fpath, out_root=".", dry_run=False): f = open(fpath, "rb") getter = util.get_getter(f, ENDIAN) get = getter.get seek = getter.seek # header fourcc, version, filecnt = get("4s2H") assert fourcc == FOURCC assert version == VERSION arc_prefix = os.path.splitext(os.path.split(fpath)[1])[0] # filelist filelist = [] for file_idx in xrange(filecnt): offset = getter.offset file_path = get("64s").rstrip("\x00") assert "\x00" not in file_path unk1, comp_size, unk2, offset = get("4I") filelist.append((file_path, offset, comp_size, unk1, unk2)) for file_path, offset, size, class_hash, crc32 in filelist: seek(offset) data = getter.get_raw(size) data_decomp = zlib.decompress(data) class_name = hash_2_classnames.get(hex(class_hash), "") ext = class_name_to_extension.get(class_name, "") if not ext: if class_name: ext = class_name elif data_decomp.startswith("<?xml"): ext = "xml" elif data_decomp.startswith("MOT"): ext = "mot" outpath = file_path final_outpath = outpath + "." + ext print hex(offset), final_outpath final_outpath = os.path.join(out_root, final_outpath) # sometimes, the same file from different arc file will collide # mostly in gui localization. It doesn't matter too much for me though. need_write = True while os.path.exists(final_outpath): print final_outpath f_old = open(final_outpath, "rb") data_old = f_old.read() f_old.close() if data_decomp == data_old: need_write = False break final_outpath += ".alias" if dry_run: need_write = False if need_write: try: util.dump_bin(data_decomp, final_outpath, mkdir=True) except IOError as e: print "hex_format", hex(class_hash) util.dump_bin(data_decomp, outpath + "_debug", mkdir=True) raise f.close()
def test_mot1(path): f = open(path, "rb") data = f.read() f.close() get = get_getter(data, ">")
def aux_parse_wmb(f): data = f.read() get = get_getter(data, ">") print "bone_count", get(48, "I")
def parse(col_path): col_f = open(col_path, "rb") col = util.get_getter(spk_f, "<") fourcc = col.get("4s").rstrip("\x00") assert fourcc == "COL"
def check_frame_size(f, log=False): data = f.read() get = get_getter(data, ">") a, b, header_size, entry_count = get(0x4, "HHII") #print a, b, base_offset = header_size entry_end = (header_size + entry_count * 0xc) now_off = entry_end last_bone_index = None last_bone_index_track_num = 0 ci = check_info() for i in xrange(entry_count): values = get(base_offset + i * 0xc, "h2b2h") if i == entry_count -1: assert values[0] == 0x7FFF break ci.check_max_track_num(values[1] + 1) if values[2] == 0: values += (get(base_offset + i * 0xc + 0x8, "f"), ) if log: print values if 3 <= values[1] < 6: ci.check_rot_value(values[5]) else: off = get(base_offset + i * 0xc + 0x8, "I") values += (off, ) if log: print values key_num = values[3] assert off == now_off if values[2] == 4: now_off += 24 + 8 * key_num ######################### # check header values # False, can't be implemented as float16 ######################### #header_values = numpy.frombuffer(buffer(data[off: off + 0x18]), # dtype=numpy.dtype(">f2")) #print header_values #assert not any(numpy.isnan(header_values)) ci.check_lerp_type4() elif values[2] == 6: now_off += 12 + 4 * key_num ci.check_max_key_num(key_num) header_values = numpy.frombuffer(buffer(data[off: off + 0xc]), dtype=numpy.dtype(">f2")) ci.check_lerp_type6(header_values) ################################################################# # check if both 0x00 and 0xff exists for a value in all keyframes # update: not really ################################################################# #max_v = [-1] * 3 #min_v = [256] * 3 #for j in xrange(key_num): # v_list = get(off + 0xc + j * 0x4 + 0x1, "BBB") # for k, v in enumerate(v_list): # max_v[k] = max(max_v[k], v) # min_v[k] = min(min_v[k], v) #for k in xrange(3): # assert (max_v[k] == 0xFF and min_v[k] == 0x00) \ # or (max_v[k] == min_v[k] and max_v[k] == 0x00) #for j in xrange(2): # assert math.fabs(header_values[j * 2]) >= math.fabs(header_values[j * 2 + 2]) # assert math.fabs(header_values[j * 2 + 1]) >= math.fabs(header_values[j * 2 + 1 + 2]) elif values[2] == 7: now_off += 12 + 6 * key_num elif values[2] == 1: now_off += 4 * key_num else: assert False, "unknown lerp type %d" % values[2] ci.check_lerp_type(values[2]) return ci