def parse_vb(self):
def conv_tangent(v):
return v / 255.0 * 2.0 - 1.0
vertices = []
vb = util.get_getter(self.vbufs[0], "<")
stride = self.vb_strides[0]
assert stride == 0x1C
for i in xrange(self.vnum):
# first vertex buffer
x, y, z = vb.get("3f")
tangent = map(conv_tangent,
vb.get("4B")) # The last element being the `sign`
for j in xrange(3):
tangent[j] *= tangent[3]
u, v = numpy.frombuffer(vb.get_raw(4), dtype=numpy.dtype("<f2"))
bone_indices = vb.get("4B")
bone_weights = vb.get("4B")
#assert sum(bone_weights) == 0xFF, "bone_weights sum=0x%x" % sum(bone_weights)
# secondary vertex buffer
vb2 = util.get_getter(self.vbufs[1], "<")
if self.vfmt2 == 10:
# Offset Sematics Format
# 0x0 TEXCOORD1 DXGI_FORMAT_R16G16_FLOAT
# 0x8 NORMAL DXGI_FORMAT_R16G16B16A16_FLOAT
vb2.skip(0x8)
nx, ny, nz, nw = numpy.frombuffer(vb2.get_raw(8),
dtype=numpy.dtype("<f2"))
elif self.vfmt2 == 7:
# Offset Sematics Format
# 0x0 TEXCOORD1 DXGI_FORMAT_R16G16_FLOAT
# 0x4 NORMAL DXGI_FORMAT_R16G16B16A16_FLOAT
vb2.skip(0x4)
nx, ny, nz, nw = numpy.frombuffer(vb2.get_raw(8),
dtype=numpy.dtype("<f2"))
elif self.vfmt2 == 11:
# Offset Sematics Format
# 0x0 TEXCOORD1 DXGI_FORMAT_R16G16_FLOAT
# 0x8 NORMAL DXGI_FORMAT_R16G16B16A16_FLOAT
# 0x10 TEXCOORD2 DXGI_FORMAT_R16G16_FLOAT
vb2.skip(0x8)
nx, ny, nz, nw = numpy.frombuffer(vb2.get_raw(8),
dtype=numpy.dtype("<f2"))
vb2.skip(0x4)
vertices.append({
"position": (x, y, z),
"normal": (nx, ny, nz),
"uv": (u, v),
"bone_indices": bone_indices,
"bone_weights": bone_weights,
})
return vertices
def get_g1mg_subchunk_data(data, schunk_type):
for chunk_data in iter_chunk(data):
get = get_getter(chunk_data, "<")
chunk_name = get(0x0, "8s")
chunk_size = get(0x8, "I")
if chunk_name.startswith(G1MG):
for subchunk_data in iter_g1mg_subchunk(chunk_data):
sub_get = get_getter(subchunk_data, "<")
if sub_get(0x0, "I") == schunk_type:
return subchunk_data
return None
def parse_vb(self):
def f(v):
return (v * 2 - 255.0) / 255.0
vertices = []
vb = util.get_getter(self.vbufs[0], "<")
stride = self.vb_strides[0]
assert stride == 0x1C
for i in xrange(self.vnum):
x, y, z = vb.get("3f")
nx, ny, nz = map(f, vb.get("3B"))
unknown = vb.get("B")
u, v = numpy.frombuffer(vb.get_raw(4), dtype=numpy.dtype("<f2"))
bone_indices = vb.get("4B")
bone_weights = vb.get("4B")
#assert sum(bone_weights) == 0xFF, "bone_weights sum=0x%x" % sum(bone_weights)
vertices.append({
"position": (x, y, z),
"normal": (nx, ny, nz),
"unknown": unknown,
"uv": (u, v),
"bone_indices": bone_indices,
"bone_weights": bone_weights,
})
return vertices
def parse_primitives(mod, dp_info): print dp_info IA_d3d10 = IA_D3D10[str(dp_info.input_layout_index)] IA_game = IA_GAME[dp_info.input_layout_index] print "input_layout_index", dp_info.input_layout_index # Parse vertices = [] # parse referrenced vertex buffer vertex_format_size = calc_vertex_format_size(IA_d3d10) getter = util.get_getter(mod.vb, "<") getter.seek(dp_info.vb_offset + vertex_format_size * dp_info.index_min) for i in xrange(dp_info.index_min, dp_info.index_max + 1, 1): vertex = parse_vertex(getter, IA_d3d10, IA_game) if mod.bone_num > 0: unnormalize_vertex(vertex, mod.inv_norm_mat) vertices.append(vertex) used_x.add(vertex["Position"][0]) used_y.add(vertex["Position"][1]) used_z.add(vertex["Position"][2]) # for k, v in sorted(vertex.iteritems()): # print k, v # print "-" * 10 return vertices
def parse(spk_fpath):
spk_f = open(spk_fpath, "rb")
spk = util.get_getter(spk_f, "<")
spk_header = spk.block(0x24)
assert spk_header.get("4s") == "SPK\x00"
assert spk_header.get("I") == 0xe11e1e22
assert spk_header.get("H") == 0x24
shader_pair_count = spk_header.get("H")
vs_count = spk_header.get("H")
ps_count = spk_header.get("H")
gs_count = spk_header.get("H")
input_layout_count = spk_header.get("H")
assert vs_count == input_layout_count
spk_header.skip(0x8)
dxbc_size = spk_header.get("I")
spk_info_size = spk_header.get("I")
assert dxbc_size + spk_info_size == spk.size
spk_header.assert_end()
spk_metadata = spk.block(spk_info_size - 0x24)
dxbc = spk.block(dxbc_size)
spk.assert_end()
spk_f.close()
pair_info_list = get_shader_pair_info(spk_metadata, shader_pair_count)
assert_shader_pair_info_list(pair_info_list, vs_count, ps_count, gs_count)
dump_vertex_shaders(spk_metadata, dxbc, 0, vs_count)
dump_pixel_shaders(spk_metadata, dxbc, 0, ps_count)
dump_shader_pair_info(spk_metadata, shader_pair_count)
return spk_header, spk_metadata, dxbc, pair_info_list
def iter_chunk(data):
get = get_getter(data, "<")
fourcc = get(0x0, "8s")
assert fourcc == G1M_0036, "invalid g1m file!"
filesize = get(0x8, "I")
assert filesize == len(data), "file size not match, file may be corrupted!"
headersize = get(0xc, "I") # usually the 1st chunk offset
unk = get(0x10, "I")
assert unk == 0x0, "this may not be a reserved field!"
chunk_count = get(0x14, "I") # usually 5 chunks but not always
off = headersize
for i in xrange(chunk_count):
chunk_name = get(off, "8s")
chunk_size = get(off + 0x8, "I")
print "chunk: %s, size: 0x%x" % (chunk_name, chunk_size)
chunk_data = data[off:off + chunk_size]
yield chunk_data
off += chunk_size
assert off == filesize, "invalid file size not match!!"
def parse(path):
f = open(path, "rb")
getter = util.get_getter(f, "<")
# read header
header = CMfxHeader()
header.read(getter)
# read string table
getter.seek(header.string_table_offset)
raw_string_table = getter.get_raw(getter.size - getter.offset)
string_table = CStringTable(raw_string_table)
# read MfxEntry
mfx_entries = []
for i, mfx_entry_offset in enumerate(header.mfx_entry_offsets):
getter.seek(mfx_entry_offset)
mfx_entry = CMfxEntry()
print i,
mfx_entry.read(getter, string_table)
mfx_entries.append(mfx_entry)
f.close()
fout = open("windbg/input_layouts2.json", "w")
input_layouts = [{}]
for mfx_entry in mfx_entries:
input_layouts.append(mfx_entry.input_layout)
while input_layouts and not input_layouts[-1]:
input_layouts.pop(-1)
json.dump(input_layouts, fout, indent=2)
fout.close()
def tex_extract(path):
f = open(path, "rb")
data = f.read()
f.close()
get = util.get_getter(data, "<")
fourcc = get(0x0, "8s")
assert fourcc == G1TG0060, "invalid or unsupported g1t file!"
file_size, header_size = get(0x8, "II")
tex_count = get(0x10, "I")
tex_off_list = get(header_size, "%dI" % tex_count, force_tuple=True)
base_off = header_size
# split
for i in xrange(tex_count):
out_path = path + ".tex%d" % i
if not os.path.exists(out_path):
off = base_off + tex_off_list[i]
end_off = file_size if (i + 1 >= tex_count) else (
base_off + tex_off_list[i + 1])
tex_blk = data[off:end_off]
fout = open(out_path, "wb")
fout.write(tex_blk)
fout.close()
# convert
for i in xrange(tex_count):
out_path = path + ".tex%d" % i
f = open(out_path, "rb")
tex_blk = f.read()
f.close()
print "conv texture %d" % i
conv_tex(tex_blk, out_path)
def parse_g1mg_subchunk_0x10005(schunk_data):
fvf_list = []
get = get_getter(schunk_data, "<")
schunk_type, schunk_size = get(0x0, "2I")
fvf_count = get(0x8, "I")
log("fvf count", fvf_count, lv=0)
off = 0xc
tot_vb_ref_list = []
for i in xrange(fvf_count):
log("fvf %d" % i, lv=0)
vb_ref_count = get(off + 0x0, "I")
vb_ref_list = get(off + 0x4, "%dI" % vb_ref_count, force_tuple=True)
tot_vb_ref_list.extend(vb_ref_list)
attr_count = get(off + 0x4 + vb_ref_count * 0x4, "I")
log("off = 0x%x, attr_count: %d, vb_ref_list:" % (off, attr_count),
vb_ref_list,
lv=0)
off += 0x8 + vb_ref_count * 0x4
attrs = []
for j in xrange(attr_count):
vb_ref_idx, offset = get(off + 0x0, "2H")
assert 0 <= vb_ref_idx < vb_ref_count
data_type = get(off + 0x4, "H")
sematics = get(off + 0x6, "H")
log("\tvb_ref_idx=%d, off=0x%x, datatype=0x%x__%s, sematics=0x%x__"\
"%s" % (vb_ref_idx, offset, data_type,
DATA_TYPE_MAP.get(data_type, "_UNK_"), sematics,
SEMATIC_NAME_MAP.get(sematics, "_UNK_")), lv=0)
off += 0x8
attrs.append(
(offset, sematics, data_type, vb_ref_idx, vb_ref_list))
fvf_list.append(attrs)
assert tot_vb_ref_list == range(len(tot_vb_ref_list))
return {"fvf_list": fvf_list}
def do_test_isize(fpath): fp = open(fpath, "rb") wmb = util.get_getter(fp, "<") res = parse_isize(wmb) fp.close() if res: print fpath
def extract(path):
f = open(path, "rb")
data = f.read()
f.close()
get = get_getter(data, "<")
fourcc = get(0x0, "4s")
assert fourcc == EARC, "invalid EARC file!"
unk = get(0x4, "I")
if unk != 0x01:
assert False, "warning: not 0x01"
data_size = get(0x8, "I")
header_size = get(0xc, "I")
filelist_size = get(0x10, "I")
file_count = get(0x14, "I")
unk = get(0x18, "I")
# assertion fails on PC version
# if unk != 1:
# assert False, "warning: not 1"
fixed_offset = get(0x1c, "I")
root = os.path.splitext(path)[0]
if not os.path.exists(root):
os.mkdir(root)
base_offset = 0x1c
last_off = None
for i in xrange(file_count):
off, size, filename = get(base_offset + i * 0x38, "2I48s")
filename = filename.rstrip("\x00")
# print "index: 0x%x raw_off: 0x%x" % (i, off)
# we should always use calculated offset for extracting file
# offset in the file will cause empty gap in memory, which is reserved for thoese 'dummy'(I guess)
off = (off - fixed_offset + header_size + filelist_size)
if last_off is not None and off != last_off:
off = last_off
end_off = off + size
last_off = end_off
# print "%03d: %s @0x%x -> 0x%x" % (i, filename, off, end_off)
assert filename == "dummy" or size != 0, "empty file always use name 'dummy'"
if size == 0:
continue
out_path = os.path.join(root, filename)
fout = open(out_path, "wb")
fout.write(data[off: off + size])
fout.close()
assert end_off == len(data), "file size not invalid!! 0x%x vs 0x%x" % (end_off, len(data))
def _parse_chunck(in_path, prefix, parse_func, *args):
fin = open(in_path, "rb")
data = fin.read()
fin.close()
for chunk_data in iter_chunk(data):
get = get_getter(chunk_data, "<")
chunk_name = get(0x0, "8s")
chunk_size = get(0x8, "I")
if chunk_name.startswith(prefix):
return parse_func(chunk_data, *args)
def handle_wmb(fpath, out_dir):
fp = open(fpath, "rb")
wmb = util.get_getter(fp, "<")
wmb = wmb_parser.parse(wmb)
fp.close()
wmb_parser.dump_wmb(
wmb,
outpath=os.path.join(out_dir,
os.path.split(fpath)[1].replace(".wmb", ".gtb"))
)
def handle_dds(wtp_path, wta_path, out_dir): if wta_path: fp = open(wta_path, "rb") wta_reader = util.get_getter(fp, "<") wta = wta_parser.parse(wta_reader) fp.close() tex_hashes = wta.texture_hashes print tex_hashes else: tex_hashes = [] splitdds.split(wtp_path, tex_hashes, out_dir)
def parse_g1mm(data):
get = get_getter(data, "<")
fourcc = get(0x0, "8s")
assert fourcc == G1MM0020, "invalid fourcc!!"
chunk_size = get(0x8, "I")
assert chunk_size == len(data), "ok"
mat_count = get(0xc, "I")
for i in xrange(mat_count):
mat = get(0x10 + 0x40 * i, "16f")
for j in xrange(4):
print mat[j * 4:(j + 1) * 4]
def collect_version(path):
version_dict = {}
for version in (0x20160116, 0x20151123, 0x20151001, 0x10000):
version_dict[version] = []
for fpath in util.iter_path(path):
if fpath.endswith(".wmb"):
print "processing:", fpath
fp = open(fpath, "rb")
wmb = util.get_getter(fp, "<")
version_dict[wmb.get("I", offset=0x4)].append(os.path.split(fpath)[1])
fp.close()
return version_dict
def convert(fpath):
fp = open(fpath, "rb")
bxm_reader = util.get_getter(fp, ">")
node_infos, keyvals = parse(bxm_reader)
fp.close()
root = gen_xml(0, node_infos, keyvals)
xml_str = etree.tostring(root, pretty_print=True, encoding="UTF-8")
fp = open(fpath.replace(".bxm", ".xml"), "w")
print fp
fp.write(xml_str)
fp.close()
def parse(data):
get = get_getter(data, ">")
off = 0x0
i = 0
while off < len(data):
str_len = get(off, "b")
if str_len == -1:
break
str_text = data[off + 1:off + 1 + str_len]
off += 1 + str_len
print "%3d:" % i, str_text
i += 1
def parse_g1mg_subchunk_0x10001(schunk_data):
dump_chunk = True
get = get_getter(schunk_data, "<")
schunk_type, schunk_size = get(0x0, "2I")
entry_count = get(0x8, "I")
reserved = get(0xc, "I")
assert reserved == 0x100
assert len(schunk_data) == 0x10 + entry_count * 0x40
log("entry_count=%d" % entry_count, lv=0)
# entry size == 0x40, but not matrix
if dump_chunk:
dump_data("g1mg_0x10001.bin", schunk_data)
def test(self):
vb0 = util.get_getter(self.vbufs[0], "<")
def f(v):
return (v * 2 - 255.0) / 255.0
for i in xrange(self.vnum):
vb0.seek(i * self.vb_strides[0])
vb0.skip(0xc) # position
normal = map(f, vb0.get(
"3B")) # mostly normalized, with some custom normals ...
unk0 = vb0.get("B")
vb0.skip(0xc) # uv, bone indices, bone weights
assert unk0 in (0x0, 0xff)
def parse(path):
load_input_layouts("windbg/input_layouts.json", "windbg/input_layouts2.json")
f = open(path, "rb")
getter = util.get_getter(f, "<")
model = CModel()
model.read(getter)
f.close()
model.dump(path.replace(".mod", ".gtb"))
print_bounding_box_check(model)
def get_vertex_data_by_datatype(data, offset, datatype): get = get_getter(data, "<") if datatype == "float": return get(offset, "f") elif datatype == "float2": return get(offset, "2f") elif datatype == "float3": return get(offset, "3f") elif datatype == "float4": return get(offset, "4f") elif datatype == "int4": return get(offset, "4B") elif datatype == "RGBA": return hex(get(offset, "I")) assert False, "impossible"
def test_vdata_ex(path):
f = open(path, "rb")
data = f.read()
f.close()
get = get_getter(data, ">")
vert_num = get(0xc, "I")
stride = 0x8
offset_beg = get(0x1c, "I")
offset_end = offset_beg + vert_num * stride
for offset in xrange(offset_beg, offset_end, stride):
c = get(offset, "I")
u1, v1 = numpy.frombuffer(buffer(data[offset + 0x4: offset + 0x8]), dtype=numpy.dtype(">f2"))
print "color:ARGB = 0x%08x, u1 = %.2f, v2 = %.2f" % (c, u1, v1)
def print_frame_info_0x4(data, offset, n):
get = get_getter(data, ">")
unk_header_size = 24
for i in xrange(3):
unk_f0 = get(offset + i * 0x8, "f")
print ("%.4f\t\t" % unk_f0), hex_format(data[offset + i * 0x8 + 0x4: offset + i * 0x8 + 0x8])
offset += unk_header_size
for i in xrange(n):
base_offset = offset + i * 0x8
unk_index, unk1, unk2, unk3 = get(base_offset, "4H")
base_offset += 0x2
unk_floats = numpy.frombuffer(buffer(data[base_offset : base_offset + 0x6]),
dtype=numpy.dtype(">f2"))
print ("F 0x%02x:\t\t" % unk_index), hex(unk1), hex(unk2), hex(unk3)
print
def parse_bone_names(path):
f = open(path, "rb")
data = f.read()
f.close()
get = get_getter(data, ">")
names = []
off = 0
while off < len(data):
str_len = get(off, "b")
if str_len == -1:
break
names.append(data[off + 1:off + 1 + str_len])
off += 1 + str_len
return names
def parse(lmt_path, out_path="objs/motion.gtba"):
f = open(lmt_path, "rb")
getter = util.get_getter(f, "<")
lmt = LMT()
lmt.read(getter)
f.close()
if EXPORT_SEPARATE_FILE:
for motion_i in xrange(len(lmt.motion_list)):
dump_single(lmt, motion_i, out_path.replace(".gtba", "_%d.gtba" % motion_i))
else:
dump_all(lmt, out_path)
return lmt
def iter_g1mg_subchunk(data):
get = get_getter(data, "<")
chunk_name = get(0x0, "8s")
chunk_size = get(0x8, "I")
platform = get(0xc, "4s")
assert platform == "DX9\x00", "platform error!"
unk = get(0x10, "I")
assert unk == 0x0, "null padding value!"
bbox = get(0x14, "6f") # (xmin, ymin, zmin, xmax, ymax, zmax)
print "bounding box:", bbox
schunk_count = get(0x2c, "I")
off = 0x30
for i in xrange(schunk_count):
schunk_type, schunk_size = get(off, "2I")
yield data[off:off + schunk_size]
off += schunk_size
def parse_g1mg_subchunk_0x10006(schunk_data):
dump_chunk = False
get = get_getter(schunk_data, "<")
schunk_type, schunk_size = get(0x0, "2I")
entry_count = get(0x8, "I")
off = 0xc
log("entry_count=%d" % entry_count, lv=0)
for entry_idx in xrange(entry_count):
item_count = get(off, "I")
mat_ref_idx, unk0, unk1, unk2, joint_map_idx = get(off + 0x4, "IHHHH")
assert unk0 == 0x8000 and unk2 == 0x8000
#count(locals(), "unk1")
off += 0x4 + item_count * 0xc
assert off == len(schunk_data)
if dump_chunk:
dump_data("g1mg_0x10006.bin", schunk_data)
def parse_g1mg_subchunk_0x10002(schunk_data):
log("========", lv=1)
log("materials", lv=1)
log("========", lv=1)
get = get_getter(schunk_data, "<")
schunk_type, schunk_size = get(0x0, "2I")
mat_count = get(0x8, "I")
# dump_data("g1mg_0x10002.bin", schunk_data)
off = 0xc
material_list = []
for mat_idx in xrange(mat_count):
unk0 = get(off + 0x0, "I")
assert unk0 == 0
tex_count = get(off + 0x4, "I")
unk1, unk2 = get(off + 0x8, "Ii")
unk1_equal_tex_count = tex_count == unk1
count(locals(), "unk1_equal_tex_count")
log("mat %d, tex_count %d, unk1=%d, unk2=%d, unk1_equal_tex_count=%d" %
(mat_idx, tex_count, unk1, unk2, unk1_equal_tex_count),
lv=1)
assert 1 <= unk1 <= 7
assert unk2 == 1 or unk2 == -1
off += 0x10
material = {"texture_count": tex_count, "textures": []}
material_list.append(material)
for tex_idx in xrange(tex_count):
tex_identifier = get(off + 0x0, "H")
uv_chnl_idx, unk6 = get(off + 0x2, "HH")
unk3, unk4, unk5 = get(off + 0x6, "3H")
count(locals(), "unk6")
assert 0 <= unk3 <= 2
assert unk4 == 4
assert unk5 == 4
assert 0 <= uv_chnl_idx <= 4, "works for this game!"
off += 0xc
log("tex_idx = %d, uv_channel_idx = %d, unk6 = %d, unk3 = %d, unk4 = %d, unk5 = %d"
% (tex_identifier, uv_chnl_idx, unk6, unk3, unk4, unk5),
lv=1)
material["textures"].append([tex_identifier, uv_chnl_idx])
log("")
return {"material_list": material_list}
def parse(f): data = f.read() get = get_getter(data, ">") FOURCC = get(0x0, "4s") assert FOURCC == "mot\x00", "invalid mot file" a, b, header_size, entry_count = get(0x4, "HHII") assert header_size == 0x10, "header_size != 0x10" print "%d, 0x%x, %d" % (a, b, get(0x4, "I")) entry_end = (header_size + entry_count * 0xc) print "entry_num = %d, entry_end = %d" % (entry_count, entry_end) base_offset = header_size last_bone_index = None last_frame_index = None now_off = entry_end offset_list = [] for i in xrange(entry_count): values = get(base_offset + i * 0xc, "h2b2h") bone_index = values[0] frame_index = values[1] if i == entry_count - 1: break if bone_index != last_bone_index: print else: assert frame_index >= last_frame_index int_impl = get(base_offset + i * 0xc + 0x8, "I") float_impl = get(base_offset + i * 0xc + 0x8, "f") if values[2] == 0: v = float_impl print_track_header(values[:5] + (v, )) else: v = int_impl offset_list.append(v) print_track_header(values[:5] + (v, )) #assert now_off == v, "expect off=%d, off=%d" % (now_off, v) f = 1 now_off += f * (12 + 4 * values[3]) if values[2] == 4: print_frame_info_0x4(data, offset_list[-1], values[3]) elif values[2] == 6: print_frame_info_0x6(data, offset_list[-1], values[3]) elif values[2] == 1: print_frame_info_0x1(data, offset_list[-1], values[3]) last_bone_index = bone_index last_frame_index = frame_index
def parse_g1mg_subchunk_0x10007(schunk_data):
print "index buffer block"
get = get_getter(schunk_data, "<")
schunk_type, schunk_size = get(0x0, "2I")
ib_count = get(0x8, "I")
off = 0xc
index_buffer_list = []
for j in xrange(ib_count):
index_count, b, c = get(off, "3I")
try:
index_buffer_list.append(get(off + 0xc, "%dH" % index_count))
except struct.error, e:
dump_data("g1mg_0x10007.bin", schunk_data)
raise e
off += 0xc + index_count * 2
print "%d => index_count: %d, 0x%x" % (j, index_count, b)
def parse_g1mg_subchunk_0x10004(schunk_data):
get = get_getter(schunk_data, "<")
schunk_type, schunk_size = get(0x0, "2I")
mesh_seg_count = get(0x8, "I")
off = 0xc
vertex_buffer_list = []
for j in xrange(mesh_seg_count):
unk1, fvf_size, vcount, unk2 = get(off, "4I")
print "%d => mesh segment @ offset=0x%x, vcount=%d, fvf_size=0x%x, unk1=%d, unk2=%d" % (
j, off, vcount, fvf_size, unk1, unk2)
assert unk1 == 0 and (unk2 == 0 or unk2 == 1), "unknown values!!"
vertex_buffer = (fvf_size, vcount, unk2, off + 0x10, schunk_data)
vertex_buffer_list.append(vertex_buffer)
off += 0x10 + vcount * fvf_size
return {"vertex_buffer_list": vertex_buffer_list}
def do_test(fpath):
print "processing:", fpath
fp = open(fpath, "rb")
wmb = util.get_getter(fp, "<")
wmb = parse(wmb)
fp.close()
if args.dry_run:
return
if args.format == "gtb":
dump_wmb(wmb, outpath=fpath.replace(".wmb", ".gtb"))
else:
for lodlv in xrange(DUMP_MAX_LOD + 1):
lod_info = wmb.lod_infos[lodlv]
for submesh_idx in xrange(lod_info.submesh_start, lod_info.submesh_start + lod_info.submesh_num):
outpath = lod_info.name + "_" + str(submesh_idx) + ".obj"
dump_submesh(wmb.submesh_infos[submesh_idx], wmb.geo_buffers, outpath)
def parse_g1mg(data):
g1mg = {}
for schunk_data in iter_g1mg_subchunk(data):
get = get_getter(schunk_data, "<")
schunk_type, schunk_size = get(0x0, "2I")
print "chunk_type=0x%x, chunk_size=0x%x" % (schunk_type, schunk_size)
assert schunk_type in xrange(
0x10001,
0x1000A), "not recognized schunk type !! 0x%x" % schunk_type
handler = G1MG_SUBCHUNK_HANDLER.get(schunk_type)
if handler is None:
continue
ret = handler(schunk_data)
if ret is not None:
g1mg.update(ret)
return g1mg
def extract_g1hp(g1hp_data): print "extracting" get = get_getter(g1hp_data, ">") fourcc = get(0x0, "8s") assert fourcc == "G1HP0010", "invalid fourcc" chunk_size = get(0x8, "I") assert len(g1hp_data) - chunk_size < 0x10, "invalid chunk size! 0x%x vs 0x%x" % (len(g1hp_data), chunk_size) unk0 = get(0xc, "I") g1m_count, unk1 = get(0x10, "2H") g1m_offset_list = get(0x14, "%dI" % g1m_count) print "offset_list", g1m_offset_list g1m_data = [] for i in xrange(g1m_count): off = g1m_offset_list[i] g1m_size = get(off + 0x8, "I") g1m_data.append(g1hp_data[off: off + g1m_size]) return g1m_data
def dump_obj(in_path, out_path):
fin = open(in_path, "rb")
data = fin.read()
fin.close()
obj_text = ""
for chunk_data in iter_chunk(data):
get = get_getter(chunk_data, "<")
chunk_name = get(0x0, "8s")
chunk_size = get(0x8, "I")
if chunk_name.startswith(G1MG):
g1mg = parse_g1mg(chunk_data)
obj_text = g1m_export.export_obj(g1mg)
break
if obj_text:
fout = open(out_path, "w")
fout.write(obj_text)
fout.close()
return None
def extract(data, path):
g1m_data = []
get = get_getter(data, ">")
fourcc = get(0x0, "8s")
assert fourcc == "G1H_0020", "invalid fourcc"
file_size = get(0x8, "I")
assert len(data) == file_size, "file size not match!"
unk0, g1hp_chunk_count = get(0xc, "2H")
assert unk0 == 0x10, "hey, guess wrong, this value has special meaning"
g1hp_chunk_offset_list = get(0x10, "%dI" % g1hp_chunk_count)
for i in xrange(g1hp_chunk_count):
off = g1hp_chunk_offset_list[i]
chunk_size = get(off + 0x8, "I")
g1m_data = extract_g1hp(data[off: off + chunk_size])
for j in xrange(len(g1m_data)):
dump_data(path.replace(".g1h", "_%d_%d.g1m" % (i, j)), g1m_data[j])
def parse_g1mg_subchunk_0x10003(schunk_data):
log("================", lv=0)
log("uniforms", lv=0)
log("================", lv=0)
dump_chunk = False
get = get_getter(schunk_data, "<")
schunk_type, schunk_size = get(0x0, "2I")
uniform_blk_cnt = get(0x8, "I")
offset = 0xc
for uniform_blk_idx in xrange(uniform_blk_cnt):
uniform_cnt = get(offset + 0x0, "I")
log("\nuniform block %d: uniform_num=%d" %
(uniform_blk_idx, uniform_cnt),
lv=0)
offset += 0x4
for uniform_idx in xrange(uniform_cnt):
tot_len, name_len = get(offset, "2I")
reserved0, datatype, reserved1 = get(offset + 0x8, "I2H")
assert reserved0 == 0 and reserved1 == 1
name = get(offset + 0x10, "%ds" % name_len).rstrip("\x00")
rem_size = tot_len - 0x10 - name_len
if 1 <= datatype <= 4:
vec_size = datatype
assert rem_size == vec_size * 0x4
values = get(offset + 0x10 + name_len,
"%df" % vec_size,
force_tuple=True)
values_string = ",".join(["%.4f" % v for v in values])
elif datatype == 5:
assert rem_size == 4
values = get(offset + 0x10 + name_len, "4B", force_tuple=True)
values_string = ",".join(["%d" % v for v in values])
else:
assert False
log("\tuniform: %s, values=%s, datatype=%d" %
(name, values_string, datatype),
lv=0)
offset += tot_len
if dump_chunk:
dump_data("g1mg_0x10003.bin", schunk_data)
def unpack_file(fname):
f = open(fname, "rb")
data = f.read()
f.close()
get = get_getter(data, ">")
FOURCC = get(0x0, "4s")
assert FOURCC == "DAT\x00"
file_count = get(0x4, "I")
blk_file_off_off, blk_ext_off, blk_filename_off, blk_file_size_off = get(0x8, "IIII")
dummys = get(0x18, "II")
assert not any(dummys)
# basicly no use
ext_list = []
offset = blk_ext_off
for i in xrange(file_count):
term_off = data.find("\x00", offset)
ext = data[offset: term_off]
offset = term_off + 1
ext_list.append(ext)
# create folders to put unpacked files in
folder = os.path.splitext(fname)[0]
if not os.path.isdir(folder):
os.mkdir(folder)
# extract files
filename_len = get(blk_filename_off, "I")
for i in xrange(file_count):
name = get(blk_filename_off + 0x4 + i * filename_len, "%ds" % filename_len).rstrip("\x00")
file_off = get(blk_file_off_off + i * 0x4, "I")
file_size = get(blk_file_size_off + i * 0x4, "I")
if file_size > 0:
file_data = data[file_off: file_off + file_size]
file_path = os.path.join(folder, name)
fout = open(file_path, "wb")
fout.write(file_data)
fout.close()
def test_vf(path):
f = open(path, "rb")
data = f.read()
f.close()
get = get_getter(data, ">")
vert_num = get(0xc, "I")
stride = 0x20
offset_beg = get(0x18, "I")
offset_end = offset_beg + vert_num * stride
for offset in xrange(offset_beg, offset_end, stride):
# pos
x, y, z = get(offset, "fff")
# uv
u, v = numpy.frombuffer(buffer(data[offset + 0xc: offset + 0x10]), dtype=numpy.dtype(">f2"))
# bone_indices
bone_indices = get(offset + 0x18, "BBBB")
# bone_weights
bone_weights = get(offset + 0x1c, "BBBB")
n1, n2, n3, n4 = get(offset + 0x10, "HHHH")
u1, v1 = numpy.frombuffer(buffer(data[offset + 0x10: offset + 0x14]), dtype=numpy.dtype(">f2"))
u2, v2 = numpy.frombuffer(buffer(data[offset + 0x14: offset + 0x18]), dtype=numpy.dtype(">f2"))
nx, ny = get(offset + 0x10, "ff")
#print "test 0x%04x, 0x%04x, 0x%04x, 0x%04x" % (n1, n2, n3, n4)
# wrong
#print "test2 %.2f, %.2f, %.2f, %.2f" % (u1, v1, u2, v2)
#print "uv1", u1, v1
#print "uv2", u2, v2
# wrong
#print "test3 %f, %f" % (nx, ny)
print "0x%08x 0x%08x" % get(offset + 0x10, "II")
def parse(path):
with open(path, "rb") as f:
data = f.read()
getter = util.get_getter(data, "<")
header = getter.block(0x18)
fourcc = header.get("4s")
assert fourcc == "XFS\x00"
unk0 = header.get("H")
unk1 = header.get("H")
unk_cnt = header.get("I")
assert unk0 & 0x7FFF == 0xF
subdata_cnt = header.get("I", offset=0x10)
subdatablock_sz = header.get("I", offset=0x14)
subdatablock = getter.block(subdatablock_sz)
off_lst = subdatablock.get("%dI" % subdata_cnt, force_tuple=True)
xml_defs = []
for i, off in enumerate(off_lst):
if i + 1 < len(off_lst):
sz = off_lst[i + 1] - off
else:
sz = subdatablock_sz - off
subdatablock.seek(off)
subdata = subdatablock.block(sz)
def_ = parse_subdata(subdatablock, subdata)
xml_defs.append(def_)
n = getter.get("I")
assert ((n >> 1) & 0x7FFF) != 0x7FFF
assert n == 1
sz = getter.get("I")
blk = getter.block(sz - 4)
parse_object(xml_defs, 0, blk)
def unpack(fpath, out_root=".", dry_run=False):
f = open(fpath, "rb")
getter = util.get_getter(f, ENDIAN)
get = getter.get
seek = getter.seek
# header
fourcc, version, filecnt = get("4s2H")
assert fourcc == FOURCC
assert version == VERSION
arc_prefix = os.path.splitext(os.path.split(fpath)[1])[0]
# filelist
filelist = []
for file_idx in xrange(filecnt):
offset = getter.offset
file_path = get("64s").rstrip("\x00")
assert "\x00" not in file_path
unk1, comp_size, unk2, offset = get("4I")
filelist.append((file_path, offset, comp_size, unk1, unk2))
for file_path, offset, size, class_hash, crc32 in filelist:
seek(offset)
data = getter.get_raw(size)
data_decomp = zlib.decompress(data)
class_name = hash_2_classnames.get(hex(class_hash), "")
ext = class_name_to_extension.get(class_name, "")
if not ext:
if class_name:
ext = class_name
elif data_decomp.startswith("<?xml"):
ext = "xml"
elif data_decomp.startswith("MOT"):
ext = "mot"
outpath = file_path
final_outpath = outpath + "." + ext
print hex(offset), final_outpath
final_outpath = os.path.join(out_root, final_outpath)
# sometimes, the same file from different arc file will collide
# mostly in gui localization. It doesn't matter too much for me though.
need_write = True
while os.path.exists(final_outpath):
print final_outpath
f_old = open(final_outpath, "rb")
data_old = f_old.read()
f_old.close()
if data_decomp == data_old:
need_write = False
break
final_outpath += ".alias"
if dry_run:
need_write = False
if need_write:
try:
util.dump_bin(data_decomp, final_outpath, mkdir=True)
except IOError as e:
print "hex_format", hex(class_hash)
util.dump_bin(data_decomp, outpath + "_debug", mkdir=True)
raise
f.close()
def test_mot1(path): f = open(path, "rb") data = f.read() f.close() get = get_getter(data, ">")
def aux_parse_wmb(f): data = f.read() get = get_getter(data, ">") print "bone_count", get(48, "I")
def parse(col_path):
col_f = open(col_path, "rb")
col = util.get_getter(spk_f, "<")
fourcc = col.get("4s").rstrip("\x00")
assert fourcc == "COL"
def check_frame_size(f, log=False):
data = f.read()
get = get_getter(data, ">")
a, b, header_size, entry_count = get(0x4, "HHII")
#print a, b,
base_offset = header_size
entry_end = (header_size + entry_count * 0xc)
now_off = entry_end
last_bone_index = None
last_bone_index_track_num = 0
ci = check_info()
for i in xrange(entry_count):
values = get(base_offset + i * 0xc, "h2b2h")
if i == entry_count -1:
assert values[0] == 0x7FFF
break
ci.check_max_track_num(values[1] + 1)
if values[2] == 0:
values += (get(base_offset + i * 0xc + 0x8, "f"), )
if log:
print values
if 3 <= values[1] < 6:
ci.check_rot_value(values[5])
else:
off = get(base_offset + i * 0xc + 0x8, "I")
values += (off, )
if log:
print values
key_num = values[3]
assert off == now_off
if values[2] == 4:
now_off += 24 + 8 * key_num
#########################
# check header values
# False, can't be implemented as float16
#########################
#header_values = numpy.frombuffer(buffer(data[off: off + 0x18]),
# dtype=numpy.dtype(">f2"))
#print header_values
#assert not any(numpy.isnan(header_values))
ci.check_lerp_type4()
elif values[2] == 6:
now_off += 12 + 4 * key_num
ci.check_max_key_num(key_num)
header_values = numpy.frombuffer(buffer(data[off: off + 0xc]),
dtype=numpy.dtype(">f2"))
ci.check_lerp_type6(header_values)
#################################################################
# check if both 0x00 and 0xff exists for a value in all keyframes
# update: not really
#################################################################
#max_v = [-1] * 3
#min_v = [256] * 3
#for j in xrange(key_num):
# v_list = get(off + 0xc + j * 0x4 + 0x1, "BBB")
# for k, v in enumerate(v_list):
# max_v[k] = max(max_v[k], v)
# min_v[k] = min(min_v[k], v)
#for k in xrange(3):
# assert (max_v[k] == 0xFF and min_v[k] == 0x00) \
# or (max_v[k] == min_v[k] and max_v[k] == 0x00)
#for j in xrange(2):
# assert math.fabs(header_values[j * 2]) >= math.fabs(header_values[j * 2 + 2])
# assert math.fabs(header_values[j * 2 + 1]) >= math.fabs(header_values[j * 2 + 1 + 2])
elif values[2] == 7:
now_off += 12 + 6 * key_num
elif values[2] == 1:
now_off += 4 * key_num
else:
assert False, "unknown lerp type %d" % values[2]
ci.check_lerp_type(values[2])
return ci
