def find_toolong_bonemorph(pmx: pmxstruct.Pmx): # check for morphs with JP names that are too long and will not be successfully saved/loaded with VMD files # for each morph, convert from string to bytes encoding to determine its length # also checks that bone/morph names can be stored in shift_jis for VMD usage core.set_encoding("shift_jis") toolong_list_bone = [] failct = 0 for d,b in enumerate(pmx.bones): try: bb = core.encode_string_with_escape(b.name_jp) if len(bb) > 15: toolong_list_bone.append("%d[%d]" % (d, len(bb))) except UnicodeEncodeError as e: core.MY_PRINT_FUNC("Bone %d" % d) # note: UnicodeEncodeError.reason has been overwritten with the string I was trying to encode, other fields unchanged newerrstr = "%s: '%s' codec cannot encode char '%s' within string '%s'" % ( e.__class__.__name__, e.encoding, e.reason[e.start:e.end], e.reason) core.MY_PRINT_FUNC(newerrstr) failct += 1 toolong_list_morph = [] for d,m in enumerate(pmx.morphs): try: mb = core.encode_string_with_escape(m.name_jp) if len(mb) > 15: toolong_list_morph.append("%d[%d]" % (d, len(mb))) except UnicodeEncodeError as e: core.MY_PRINT_FUNC("Morph %d" % d) # note: UnicodeEncodeError.reason has been overwritten with the string I was trying to encode, other fields unchanged newerrstr = "%s: '%s' codec cannot encode char '%s' within string '%s'" % ( e.__class__.__name__, e.encoding, e.reason[e.start:e.end], e.reason) core.MY_PRINT_FUNC(newerrstr) failct += 1 if failct: core.MY_PRINT_FUNC("WARNING: found %d JP names that cannot be encoded with SHIFT-JIS, this will cause MMD to behave strangely. Please replace the bad characters in the strings printed above!" % failct) return toolong_list_bone, toolong_list_morph
def write_vmd(vmd_filename: str, vmd: vmdstruct.Vmd, moreinfo=False): vmd_filename_clean = core.get_clean_basename(vmd_filename) + ".vmd" # recives object (header, boneframe_list, morphframe_list, camframe_list, lightframe_list, shadowframe_list, ikdispframe_list) # first, verify that the data is valid before trying to write vmd.validate() # assumes the calling function already verified correct file extension core.MY_PRINT_FUNC("Begin encoding VMD file '%s'" % vmd_filename_clean) core.set_encoding("shift_jis") core.print_progress_oneline(0) # this is where sorting happens, if it happens if GUARANTEE_FRAMES_SORTED: # bones & morphs: primarily sorted by NAME, with FRAME# as tiebreaker. the second sort is the primary one. vmd.boneframes.sort(key=lambda x: x.f) # frame# vmd.boneframes.sort(key=lambda x: x.name) # name vmd.morphframes.sort(key=lambda x: x.f) vmd.morphframes.sort(key=lambda x: x.name) # all of these only sort by frame number. vmd.camframes.sort(key=lambda x: x.f) # frame# vmd.lightframes.sort(key=lambda x: x.f) vmd.shadowframes.sort(key=lambda x: x.f) vmd.ikdispframes.sort(key=lambda x: x.f) global ENCODE_PERCENT_BONE global ENCODE_PERCENT_MORPH # cam is not included cuz a file contains only bone+morph OR cam total_bone = len(vmd.boneframes) * ENCODE_FACTOR_BONE total_morph = len(vmd.morphframes) * ENCODE_FACTOR_MORPH ALLENCODE = total_bone + total_morph if ALLENCODE == 0: ALLENCODE = 1 # just a bandaid to avoid zero-div error when writing empty VMD ENCODE_PERCENT_BONE = total_bone / ALLENCODE ENCODE_PERCENT_MORPH = total_morph / ALLENCODE # arg "vmd" is the same structure created by "parse_vmd()" # assume the object is perfect, no sanity-checking needed, it will all be done when parsing the text input output_bytes = bytearray() output_bytes += encode_vmd_header(vmd.header, moreinfo) output_bytes += encode_vmd_boneframe(vmd.boneframes, moreinfo) output_bytes += encode_vmd_morphframe(vmd.morphframes, moreinfo) output_bytes += encode_vmd_camframe(vmd.camframes, moreinfo) output_bytes += encode_vmd_lightframe(vmd.lightframes, moreinfo) output_bytes += encode_vmd_shadowframe(vmd.shadowframes, moreinfo) output_bytes += encode_vmd_ikdispframe(vmd.ikdispframes, moreinfo) # done encoding!! # add a cheeky little binary stamp just to prove that people actually used my tool :) if APPEND_SIGNATURE: # signature to prove that this file was created with this tool output_bytes += bytes(SIGNATURE, encoding="shift_jis") core.MY_PRINT_FUNC("Begin writing VMD file '%s'" % vmd_filename_clean) core.MY_PRINT_FUNC("...total size = %s" % core.prettyprint_file_size(len(output_bytes))) core.write_bytes_to_binfile(vmd_filename, output_bytes) core.MY_PRINT_FUNC("Done writing VMD file '%s'" % vmd_filename_clean) # done with everything! return
def read_vmd(vmd_filename: str, moreinfo=False) -> vmdstruct.Vmd: vmd_filename_clean = core.get_clean_basename(vmd_filename) + ".vmd" # creates object (header, boneframe_list, morphframe_list, camframe_list, lightframe_list, shadowframe_list, ikdispframe_list) # assumes the calling function already verified correct file extension core.MY_PRINT_FUNC("Begin reading VMD file '%s'" % vmd_filename_clean) vmd_bytes = core.read_binfile_to_bytes(vmd_filename) core.MY_PRINT_FUNC("...total size = %sKB" % round(len(vmd_bytes) / 1024)) core.MY_PRINT_FUNC("Begin parsing VMD file '%s'" % vmd_filename_clean) core.reset_unpack() core.set_encoding("shift_jis") # !!!! this does eliminate all the garbage data MMD used to pack strings so this isnt 100% reversable !!! # read the bytes object and return all the data from teh VMD broken up into a list of lists # also convert things from packed formats to human-readable scales # (quaternion to euler, radians to degrees, floats to ints, etc) # also generate the bonedict and morphdict core.print_progress_oneline(0) A = parse_vmd_header(vmd_bytes, moreinfo) B = parse_vmd_boneframe(vmd_bytes, moreinfo) C = parse_vmd_morphframe(vmd_bytes, moreinfo) D = parse_vmd_camframe(vmd_bytes, moreinfo) E = parse_vmd_lightframe(vmd_bytes, moreinfo) F = parse_vmd_shadowframe(vmd_bytes, moreinfo) G = parse_vmd_ikdispframe(vmd_bytes, moreinfo) if moreinfo: core.print_failed_decodes() bytes_remain = len(vmd_bytes) - core.get_readfrom_byte() if bytes_remain != 0: # padding with my SIGNATURE is acceptable, anything else is strange leftover = vmd_bytes[core.get_readfrom_byte():] if leftover == bytes(SIGNATURE, encoding="shift_jis"): core.MY_PRINT_FUNC( "...note: this VMD file was previously modified with this tool!" ) else: core.MY_PRINT_FUNC( "Warning: finished parsing but %d bytes are left over at the tail!" % bytes_remain) core.MY_PRINT_FUNC( "The file may be corrupt or maybe it contains unknown/unsupported data formats" ) core.MY_PRINT_FUNC(leftover) core.MY_PRINT_FUNC("Done parsing VMD file '%s'" % vmd_filename_clean) vmd = vmdstruct.Vmd(A, B, C, D, E, F, G) # this is where sorting happens, if it happens if GUARANTEE_FRAMES_SORTED: # bones & morphs: primarily sorted by NAME, with FRAME# as tiebreaker. the second sort is the primary one. vmd.boneframes.sort(key=lambda x: x.f) # frame# vmd.boneframes.sort(key=lambda x: x.name) # name vmd.morphframes.sort(key=lambda x: x.f) vmd.morphframes.sort(key=lambda x: x.name) # all of these only sort by frame number. vmd.camframes.sort(key=lambda x: x.f) # frame# vmd.lightframes.sort(key=lambda x: x.f) vmd.shadowframes.sort(key=lambda x: x.f) vmd.ikdispframes.sort(key=lambda x: x.f) return vmd
def find_shiftjis_unsupported_names(pmx: pmxstruct.Pmx, filepath: str) -> int: # checks that bone/morph names can be stored in shift_jis for VMD usage # also check the model name and the filepath core.set_encoding("shift_jis") failct = 0 print(filepath) # first, full absolute file path: try: _ = core.encode_string_with_escape(filepath) except UnicodeEncodeError as e: core.MY_PRINT_FUNC("Filepath") # note: UnicodeEncodeError.reason has been overwritten with the string I was trying to encode, other fields unchanged newerrstr = "%s: '%s' codec cannot encode char '%s' within string '%s'" % ( e.__class__.__name__, e.encoding, e.reason[e.start:e.end], e.reason) core.MY_PRINT_FUNC(newerrstr) failct += 1 # second, JP model name: try: _ = core.encode_string_with_escape(pmx.header.name_jp) except UnicodeEncodeError as e: core.MY_PRINT_FUNC("Model Name") # note: UnicodeEncodeError.reason has been overwritten with the string I was trying to encode, other fields unchanged newerrstr = "%s: '%s' codec cannot encode char '%s' within string '%s'" % ( e.__class__.__name__, e.encoding, e.reason[e.start:e.end], e.reason) core.MY_PRINT_FUNC(newerrstr) failct += 1 # third, bones for d, b in enumerate(pmx.bones): try: _ = core.encode_string_with_escape(b.name_jp) except UnicodeEncodeError as e: core.MY_PRINT_FUNC("Bone %d" % d) # note: UnicodeEncodeError.reason has been overwritten with the string I was trying to encode, other fields unchanged newerrstr = "%s: '%s' codec cannot encode char '%s' within string '%s'" % ( e.__class__.__name__, e.encoding, e.reason[e.start:e.end], e.reason) core.MY_PRINT_FUNC(newerrstr) failct += 1 # fourth, morphs for d, m in enumerate(pmx.morphs): try: _ = core.encode_string_with_escape(m.name_jp) except UnicodeEncodeError as e: core.MY_PRINT_FUNC("Morph %d" % d) # note: UnicodeEncodeError.reason has been overwritten with the string I was trying to encode, other fields unchanged newerrstr = "%s: '%s' codec cannot encode char '%s' within string '%s'" % ( e.__class__.__name__, e.encoding, e.reason[e.start:e.end], e.reason) core.MY_PRINT_FUNC(newerrstr) failct += 1 return failct
def find_toolong_bonemorph(pmx: pmxstruct.Pmx) -> (list, list): # check for morphs with JP names that are too long and will not be successfully saved/loaded with VMD files # for each morph, convert from string to bytes encoding to determine its length core.set_encoding("shift_jis") toolong_list_bone = [] for d, b in enumerate(pmx.bones): try: bb = core.encode_string_with_escape(b.name_jp) if len(bb) > 15: toolong_list_bone.append("%d[%d]" % (d, len(bb))) except UnicodeEncodeError: # if shift-jis cannot the chars in this name, then report that in find_shiftjis_unsupported_names() pass toolong_list_morph = [] for d, m in enumerate(pmx.morphs): try: mb = core.encode_string_with_escape(m.name_jp) if len(mb) > 15: toolong_list_morph.append("%d[%d]" % (d, len(mb))) except UnicodeEncodeError: # if shift-jis cannot the chars in this name, then report that in find_shiftjis_unsupported_names() pass return toolong_list_bone, toolong_list_morph
def main(moreinfo=True): # prompt PMX name core.MY_PRINT_FUNC("Please enter name of PMX input file:") input_filename_pmx = core.MY_FILEPROMPT_FUNC(".pmx") pmx = pmxlib.read_pmx(input_filename_pmx, moreinfo=moreinfo) # prompt VMD file name core.MY_PRINT_FUNC("") core.MY_PRINT_FUNC( "Please enter name of VMD motion or VPD pose file to check compatability with:" ) input_filename = core.MY_FILEPROMPT_FUNC(".vmd .vpd") if not input_filename.lower().endswith(".vpd"): # the actual VMD part isn't even used, only bonedict and morphdict vmd = vmdlib.read_vmd(input_filename, moreinfo=moreinfo) else: vmd = vpdlib.read_vpd(input_filename, moreinfo=moreinfo) bonedict = vmdlib.parse_vmd_used_dict(vmd.boneframes, frametype="bone", moreinfo=moreinfo) morphdict = vmdlib.parse_vmd_used_dict(vmd.morphframes, frametype="morph", moreinfo=moreinfo) core.MY_PRINT_FUNC("") # must use same encoding as I used when the VMD was unpacked, since the hex bytes only have meaning in that encoding core.set_encoding("shift_jis") ############################################## # check morph compatability # build list of morphs used in the dance VMD morphs_in_vmd = list(morphdict.keys()) # build list of ALL morphs in the PMX morphs_in_model = [pmxmorph.name_jp for pmxmorph in pmx.morphs] # ensure that the VMD contains at least some morphs, to prevent zero-divide error if len(morphs_in_vmd) == 0: core.MY_PRINT_FUNC( "MORPH SKIP: VMD '%s' does not contain any morphs that are used in a meaningful way." % core.get_clean_basename(input_filename)) elif len(morphs_in_model) == 0: core.MY_PRINT_FUNC( "MORPH SKIP: PMX '%s' does not contain any morphs." % core.get_clean_basename(input_filename_pmx)) else: # convert pmx-morph names to bytes # these can plausibly fail shift_jis encoding because they came from the UTF-8 pmx file morphs_in_model_b = [] for a in morphs_in_model: try: b = core.encode_string_with_escape(a) except UnicodeEncodeError as e: newerrstr = "%s: '%s' codec cannot encode char '%s' within string '%s'" % ( e.__class__.__name__, e.encoding, e.reason[e.start:e.end], e.reason) core.MY_PRINT_FUNC(newerrstr) b = bytearray() morphs_in_model_b.append(b) # convert vmd-morph names to bytes # these might be truncated but cannot fail because they were already decoded from the shift_jis vmd file morphs_in_vmd_b = [ core.encode_string_with_escape(a) for a in morphs_in_vmd ] matching_morphs = {} missing_morphs = {} # iterate over list of morphs for vmdmorph, vmdmorph_b in zip(morphs_in_vmd, morphs_in_vmd_b): # question: does "vmdmorph" match something in "morphs_in_model"? # BUT, doing comparison in bytes-space to handle escape characters: vmdmorph_b vs morphs_in_model_b # NOTE: MMD does not try to use "begins-with" matching like I had hoped/assumed, it only looks for exact matches # return list of ALL matches, this way i can raise an error if there are multiple matches # exact match modelmorphmatch_b = [ a for a in morphs_in_model_b if a == vmdmorph_b ] # copy the key,val in one of the dicts depending on results of matching attempt if len(modelmorphmatch_b) == 0: # MISS! key is the VMD morph name since that's the best clue Ive got missing_morphs[vmdmorph] = morphdict[vmdmorph] elif len(modelmorphmatch_b) == 1: # MATCH! key is the PMX morph name it matched against, since it might be a longer version wtihout escape char matching_morphs[core.decode_bytes_with_escape( modelmorphmatch_b[0])] = morphdict[vmdmorph] else: # more than 1 morph was a match!? core.MY_PRINT_FUNC( "Warning: VMDmorph '%s' matched multiple PMXmorphs, its behavior is uncertain." % vmdmorph) modelmorphmatch = [ core.decode_bytes_with_escape(a) for a in modelmorphmatch_b ] # core.MY_PRINT_FUNC(modelmorphmatch) matching_morphs[modelmorphmatch[0]] = morphdict[vmdmorph] # display results! r = "PASS" if len(matching_morphs) == len(morphs_in_vmd) else "FAIL" core.MY_PRINT_FUNC( "MORPH {}: {} / {} = {:.1%} of the morphs are supported".format( r, len(matching_morphs), len(morphs_in_vmd), len(matching_morphs) / len(morphs_in_vmd))) # if there are no missing morphs (all match), don't print anything at all if missing_morphs: if not moreinfo: core.MY_PRINT_FUNC( "For detailed list, please re-run with 'more info' enabled" ) else: # convert the dicts to lists and sort for printing # sort in-place descending by 2nd element as primary missing_morphs_list = sorted(list(missing_morphs.items()), key=core.get2nd, reverse=True) # justify the names! missing_just = core.MY_JUSTIFY_STRINGLIST( ["'" + m[0] + "'" for m in missing_morphs_list]) # re-attach the justified names to the usage numbers missing_morphs_list = list( zip(missing_just, [m[1] for m in missing_morphs_list])) core.MY_PRINT_FUNC("") core.MY_PRINT_FUNC("Unsupported morphs: name + times used") for m, num in missing_morphs_list: core.MY_PRINT_FUNC(" %s || %d" % (m, int(num))) # only print the matching morphs if there are some, and if enabled by options if matching_morphs and PRINT_MATCHING_ITEMS: matching_morphs_list = list(matching_morphs.items()) matching_morphs_list.sort( key=core.get2nd, reverse=True ) # sort in-place descending by 2nd element as primary matching_just = core.MY_JUSTIFY_STRINGLIST( ["'" + m[0] + "'" for m in matching_morphs_list]) matching_morphs_list = list( zip(matching_just, [m[1] for m in matching_morphs_list])) core.MY_PRINT_FUNC("") core.MY_PRINT_FUNC("Supported morphs: name + times used") for m, num in matching_morphs_list: core.MY_PRINT_FUNC(" %s || %d" % (m, int(num))) ############################################## # check bone compatability core.MY_PRINT_FUNC("") # build list of bones used in the dance VMD bones_in_vmd = list(bonedict.keys()) # build list of ALL bones in the PMX # first item of pmxbone is the jp name bones_in_model = [pmxbone.name_jp for pmxbone in pmx.bones] # ensure that the VMD contains at least some bones, to prevent zero-divide error if len(bones_in_vmd) == 0: core.MY_PRINT_FUNC( "BONE SKIP: VMD '%s' does not contain any bones that are used in a meaningful way." % core.get_clean_basename(input_filename)) elif len(bones_in_model) == 0: core.MY_PRINT_FUNC("BONE SKIP: PMX '%s' does not contain any bones." % core.get_clean_basename(input_filename_pmx)) else: # convert pmx-bone names to bytes # these can plausibly fail shift_jis encoding because they came from the UTF-8 pmx file bones_in_model_b = [] for a in bones_in_model: try: b = core.encode_string_with_escape(a) except UnicodeEncodeError as e: newerrstr = "%s: '%s' codec cannot encode char '%s' within string '%s'" % ( e.__class__.__name__, e.encoding, e.reason[e.start:e.end], e.reason) core.MY_PRINT_FUNC(newerrstr) b = bytearray() bones_in_model_b.append(b) # convert vmd-bone names to bytes # these might be truncated but cannot fail because they were already decoded from the shift_jis vmd file bones_in_vmd_b = [ core.encode_string_with_escape(a) for a in bones_in_vmd ] matching_bones = {} missing_bones = {} # iterate over list of bones that pass the size check for vmdbone, vmdbone_b in zip(bones_in_vmd, bones_in_vmd_b): # question: does "vmdbone" match something in "bones_in_model"? # BUT, doing comparison in bytes-space to handle escape characters: vmdbone_b vs bones_in_model_b # NOTE: MMD does not try to use "begins-with" matching like I had hoped/assumed, it only looks for exact matches # return list of ALL matches, this way i can raise an error if there are multiple matches # exact match modelbonematch_b = [a for a in bones_in_model_b if a == vmdbone_b] # copy the key,val in one of the dicts depending on results of matching attempt if len(modelbonematch_b) == 0: # MISS! key is the VMD bone name since that's the best clue Ive got missing_bones[vmdbone] = bonedict[vmdbone] elif len(modelbonematch_b) == 1: # MATCH! key is the PMX bone name it matched against, since it might be a longer version wtihout escape char matching_bones[core.decode_bytes_with_escape( modelbonematch_b[0])] = bonedict[vmdbone] else: # more than 1 bone was a match!? core.MY_PRINT_FUNC( "Warning: VMDbone '%s' matched multiple PMXbones, its behavior is uncertain." % vmdbone) modelbonematch = [ core.decode_bytes_with_escape(a) for a in modelbonematch_b ] # core.MY_PRINT_FUNC(modelbonematch) matching_bones[modelbonematch[0]] = bonedict[vmdbone] # display results! r = "PASS" if len(matching_bones) == len(bones_in_vmd) else "FAIL" core.MY_PRINT_FUNC( "BONE {}: {} / {} = {:.1%} of the bones are supported".format( r, len(matching_bones), len(bones_in_vmd), len(matching_bones) / len(bones_in_vmd))) # if there are no missing bones (all match), don't print anything at all if missing_bones: if not moreinfo: core.MY_PRINT_FUNC( "For detailed list, please re-run with 'more info' enabled" ) else: # convert the dicts to lists and sort for printing # sort in-place descending by 2nd element as primary missing_bones_list = sorted(list(missing_bones.items()), key=core.get2nd, reverse=True) # justify the names! missing_just = core.MY_JUSTIFY_STRINGLIST( ["'" + m[0] + "'" for m in missing_bones_list]) # re-attach the justified names to the usage numbers missing_bones_list = list( zip(missing_just, [m[1] for m in missing_bones_list])) core.MY_PRINT_FUNC("") core.MY_PRINT_FUNC("Unsupported bones: name + times used") for m, num in missing_bones_list: core.MY_PRINT_FUNC(" %s || %d" % (m, int(num))) # only print the matching bones if there are some, and if enabled by options if matching_bones and PRINT_MATCHING_ITEMS: matching_bones_list = list(matching_bones.items()) matching_bones_list.sort( key=core.get2nd, reverse=True ) # sort in-place descending by 2nd element as primary matching_just = core.MY_JUSTIFY_STRINGLIST( ["'" + m[0] + "'" for m in matching_bones_list]) matching_bones_list = list( zip(matching_just, [m[1] for m in matching_bones_list])) core.MY_PRINT_FUNC("") core.MY_PRINT_FUNC("Supported bones: name + times used") for m, num in matching_bones_list: core.MY_PRINT_FUNC(" %s || %d" % (m, int(num))) # NEW: among matching bones, check whether any bones have unsupported translation/rotation for bonestr in sorted(list(matching_bones.keys())): # get the bone to get whether rot/trans enabled bone = core.my_list_search(pmx.bones, lambda x: x.name_jp == bonestr, getitem=True) # get all the frames from the VMD that are relevant to this bone thisboneframes = [f for f in vmd.boneframes if f.name == bonestr] # does the VMD use rotation? probably, check anyway vmd_use_rot = any(f.rot != [0, 0, 0] for f in thisboneframes) if vmd_use_rot and not (bone.has_rotate and bone.has_enabled): # raise some sort of warning w = "Warning: supported bone '%s' uses rotation in VMD, but rotation not allowed by PMX" % bonestr core.MY_PRINT_FUNC(w) # does the VMD use translation? vmd_use_trans = any(f.pos != [0, 0, 0] for f in thisboneframes) if vmd_use_trans and not (bone.has_translate and bone.has_enabled): # raise some sort of warning w = "Warning: supported bone '%s' uses move/shift in VMD, but move/shift not allowed by PMX" % bonestr core.MY_PRINT_FUNC(w) core.MY_PRINT_FUNC("") core.MY_PRINT_FUNC("Done!") return None