def main(moreinfo=False):
# step zero: verify that Pillow exists
if Image is None:
core.MY_PRINT_FUNC("ERROR: Python library 'Pillow' not found. This script requires this library to run!")
core.MY_PRINT_FUNC("This script cannot be ran from the EXE version, the Pillow library is too large to package into the executable.")
core.MY_PRINT_FUNC("To install Pillow, please use the command 'pip install Pillow' in the Windows command prompt and then run the Python scripts directly.")
return None
# print pillow version just cuz
core.MY_PRINT_FUNC("Using Pillow version '%s'" % Image.__version__)
core.MY_PRINT_FUNC("Please enter name of PMX model file:")
input_filename_pmx = core.MY_FILEPROMPT_FUNC(".pmx")
# absolute path to directory holding the pmx
input_filename_pmx_abs = os.path.normpath(os.path.abspath(input_filename_pmx))
startpath, input_filename_pmx_rel = os.path.split(input_filename_pmx_abs)
# =========================================================================================================
# =========================================================================================================
# =========================================================================================================
# first, build the list of ALL files that actually exist, then filter it down to neighbor PMXs and relevant files
relative_all_exist_files = file_sort_textures.walk_filetree_from_root(startpath)
core.MY_PRINT_FUNC("ALL EXISTING FILES:", len(relative_all_exist_files))
# now fill "neighbor_pmx" by finding files without path separator that end in PMX
# these are relative paths tho
neighbor_pmx = [f for f in relative_all_exist_files if
(f.lower().endswith(".pmx")) and
(os.path.sep not in f) and
f != input_filename_pmx_rel]
core.MY_PRINT_FUNC("NEIGHBOR PMX FILES:", len(neighbor_pmx))
# filter down to just image files
relevant_exist_files = [f for f in relative_all_exist_files if f.lower().endswith(IMG_EXT)]
core.MY_PRINT_FUNC("RELEVANT EXISTING FILES:", len(relevant_exist_files))
# =========================================================================================================
# =========================================================================================================
# =========================================================================================================
# now ask if I care about the neighbors and read the PMXes into memory
pmx_filenames = [input_filename_pmx_rel]
if neighbor_pmx:
core.MY_PRINT_FUNC("")
info = [
"Detected %d top-level neighboring PMX files, these probably share the same filebase as the target." % len(neighbor_pmx),
"If files are moved/renamed but the neighbors are not processed, the neighbor texture references will probably break.",
"Do you want to process all neighbors in addition to the target? (highly recommended)",
"1 = Yes, 2 = No"]
r = core.MY_SIMPLECHOICE_FUNC((1, 2), info)
if r == 1:
core.MY_PRINT_FUNC("Processing target + all neighbor files")
# append neighbor PMX files onto the list of files to be processed
pmx_filenames += neighbor_pmx
else:
core.MY_PRINT_FUNC("WARNING: Processing only target, ignoring %d neighbor PMX files" % len(neighbor_pmx))
# now read all the PMX objects & store in dict alongside the relative name
# dictionary where keys are filename and values are resulting pmx objects
all_pmx_obj = {}
for this_pmx_name in pmx_filenames:
this_pmx_obj = pmxlib.read_pmx(os.path.join(startpath, this_pmx_name), moreinfo=moreinfo)
all_pmx_obj[this_pmx_name] = this_pmx_obj
# =========================================================================================================
# =========================================================================================================
# =========================================================================================================
# for each pmx, for each file on disk, match against files used in textures (case-insensitive) and replace with canonical name-on-disk
# also fill out how much and how each file is used, and unify dupes between files, all that good stuff
filerecord_list = file_sort_textures.categorize_files(all_pmx_obj, relevant_exist_files, moreinfo)
# =========================================================================================================
# =========================================================================================================
# =========================================================================================================
# DETERMINE NEW NAMES FOR FILES
# first, create a backup of the folder
# save the name, so that i can delete it if i didn't make any changes
zipfile_name = ""
if MAKE_BACKUP_ZIPFILE:
r = file_sort_textures.make_zipfile_backup(startpath, BACKUP_SUFFIX)
if not r:
# this happens if the backup failed somehow AND the user decided to quit
core.MY_PRINT_FUNC("Aborting: no files were changed")
return None
zipfile_name = r
# name used for temporary location
tempfilename = os.path.join(startpath,"temp_image_file_just_delete_me.png")
pil_cannot_inspect = 0
pil_cannot_inspect_list = []
pil_imgext_mismatch = 0
num_recompressed = 0
# list of memory saved by recompressing each file. same order/length as "image_filerecords"
mem_saved = []
# make image persistient, so I know it always exists and I can always call "close" before open
im = None
# only iterate over images that exist, obviously
image_filerecords = [f for f in filerecord_list if f.exists]
# iterate over the images
for i, p in enumerate(image_filerecords):
abspath = os.path.join(startpath, p.name)
orig_size = os.path.getsize(abspath)
# if not moreinfo, then each line overwrites the previous like a progress printout does
# if moreinfo, then each line is printed permanently
core.MY_PRINT_FUNC("...analyzing {:>3}/{:>3}, file='{}', size={} ".format(
i+1, len(image_filerecords), p.name, core.prettyprint_file_size(orig_size)), is_progress=(not moreinfo))
mem_saved.append(0)
# before opening, try to close it just in case
if im is not None:
im.close()
# open the image & catch all possible errors
try:
im = Image.open(abspath)
except FileNotFoundError as eeee:
core.MY_PRINT_FUNC("FILESYSTEM MALFUNCTION!!", eeee.__class__.__name__, eeee)
core.MY_PRINT_FUNC("os.walk created a list of all filenames on disk, but then this filename doesn't exist when i try to open it?")
im = None
except OSError as eeee:
# this has 2 causes, "Unsupported BMP bitfields layout" or "cannot identify image file"
if DEBUG:
print("CANNOT INSPECT!1", eeee.__class__.__name__, eeee, p.name)
im = None
except NotImplementedError as eeee:
# this is because there's some DDS format it can't make sense of
if DEBUG:
print("CANNOT INSPECT!2", eeee.__class__.__name__, eeee, p.name)
im = None
if im is None:
pil_cannot_inspect += 1
pil_cannot_inspect_list.append(p.name)
continue
if im.format not in IMG_TYPE_TO_EXT:
core.MY_PRINT_FUNC("WARNING: file '%s' has unusual image format '%s', attempting to continue" % (p.name, im.format))
# now the image is successfully opened!
newname = p.name
base, currext = os.path.splitext(newname)
# 1, depending on image format, attempt to re-save as PNG
if im.format not in IM_FORMAT_ALWAYS_SKIP:
# delete temp file if it still exists
if os.path.exists(tempfilename):
try:
os.remove(tempfilename)
except OSError as e:
core.MY_PRINT_FUNC(e.__class__.__name__, e)
core.MY_PRINT_FUNC("ERROR1: failed to delete temp image file '%s' during processing" % tempfilename)
break
# save to tempfilename with png format, use optimize=true
try:
im.save(tempfilename, format="PNG", optimize=True)
except OSError as e:
core.MY_PRINT_FUNC(e.__class__.__name__, e)
core.MY_PRINT_FUNC("ERROR2: failed to re-compress image '%s', original not modified" % p.name)
continue
# measure & compare file size
new_size = os.path.getsize(tempfilename)
diff = orig_size - new_size
# if using a 16-bit BMP format, re-save back to bmp with same name
is_bad_bmp = False
if im.format == "BMP":
try:
# this might fail, images are weird, sometimes they don't have the attributes i expect
if im.tile[0][3][0] in KNOWN_BAD_FORMATS:
is_bad_bmp = True
except Exception as e:
if DEBUG:
print(e.__class__.__name__, e, "BMP THING", p.name, im.tile)
if diff > (REQUIRED_COMPRESSION_AMOUNT_KB * 1024) \
or is_bad_bmp\
or im.format in IM_FORMAT_ALWAYS_CONVERT:
# if it frees up at least XXX kb, i will keep it!
# also keep it if it is a bmp encoded with 15-bit or 16-bit colors
# set p.newname = png, and delete original and move tempname to base.png
try:
# delete original
os.remove(os.path.join(startpath, p.name))
except OSError as e:
core.MY_PRINT_FUNC(e.__class__.__name__, e)
core.MY_PRINT_FUNC("ERROR3: failed to delete old image '%s' after recompressing" % p.name)
continue
newname = base + ".png"
# resolve potential collisions by adding numbers suffix to file names
# first need to make path absolute so get_unused_file_name can check the disk.
newname = os.path.join(startpath, newname)
# then check uniqueness against files on disk
newname = core.get_unused_file_name(newname)
# now dest path is guaranteed unique against other existing files
# make the path no longer absolute: undo adding "startpath" above
newname = os.path.relpath(newname, startpath)
try:
# move new into place
os.rename(tempfilename, os.path.join(startpath, newname))
except OSError as e:
core.MY_PRINT_FUNC(e.__class__.__name__, e)
core.MY_PRINT_FUNC("ERROR4: after deleting original '%s', failed to move recompressed version into place!" % p.name)
continue
num_recompressed += 1
p.newname = newname
mem_saved[-1] = diff
continue # if succesfully re-saved, do not do the extension-checking below
# if this is not sufficiently compressed, do not use "continue", DO hit the extension-checking below
# 2, if the file extension doesn't match with the image type, then make it match
# this only happens if the image was not re-saved above
if im.format in IMG_TYPE_TO_EXT and currext not in IMG_TYPE_TO_EXT[im.format]:
newname = base + IMG_TYPE_TO_EXT[im.format][0]
# resolve potential collisions by adding numbers suffix to file names
# first need to make path absolute so get_unused_file_name can check the disk.
newname = os.path.join(startpath, newname)
# then check uniqueness against files on disk
newname = core.get_unused_file_name(newname)
# now dest path is guaranteed unique against other existing files
# make the path no longer absolute: undo adding "startpath" above
newname = os.path.relpath(newname, startpath)
# do the actual rename here & now
try:
# os.renames creates all necessary intermediate folders needed for the destination
# it also deletes the source folders if they become empty after the rename operation
os.renames(os.path.join(startpath, p.name), os.path.join(startpath, newname))
except OSError as e:
core.MY_PRINT_FUNC(e.__class__.__name__, e)
core.MY_PRINT_FUNC("ERROR5: unable to rename file '%s' --> '%s', attempting to continue with other file rename operations"
% (p.name, newname))
continue
pil_imgext_mismatch += 1
p.newname = newname
continue
# these must be the same length after iterating
assert len(mem_saved) == len(image_filerecords)
# if the image is still open, close it
if im is not None:
im.close()
# delete temp file if it still exists
if os.path.exists(tempfilename):
try:
os.remove(tempfilename)
except OSError as e:
core.MY_PRINT_FUNC(e.__class__.__name__, e)
core.MY_PRINT_FUNC("WARNING: failed to delete temp image file '%s' after processing" % tempfilename)
# =========================================================================================================
# =========================================================================================================
# =========================================================================================================
# are there any with proposed renaming?
if not any(u.newname is not None for u in image_filerecords):
core.MY_PRINT_FUNC("No proposed file changes")
# if nothing was changed, delete the backup zip!
core.MY_PRINT_FUNC("Deleting backup archive")
if os.path.exists(zipfile_name):
try:
os.remove(zipfile_name)
except OSError as e:
core.MY_PRINT_FUNC(e.__class__.__name__, e)
core.MY_PRINT_FUNC("WARNING: failed to delete pointless zip file '%s'" % zipfile_name)
core.MY_PRINT_FUNC("Aborting: no files were changed")
return None
# =========================================================================================================
# =========================================================================================================
# =========================================================================================================
# finally, do the actual renaming:
# do all renaming in PMXes
file_sort_textures.apply_file_renaming(all_pmx_obj, image_filerecords, startpath, skipdiskrename=True)
# write out
for this_pmx_name, this_pmx_obj in all_pmx_obj.items():
# NOTE: this is OVERWRITING THE PREVIOUS PMX FILE, NOT CREATING A NEW ONE
# because I make a zipfile backup I don't need to feel worried about preserving the old version
output_filename_pmx = os.path.join(startpath, this_pmx_name)
# output_filename_pmx = core.get_unused_file_name(output_filename_pmx)
pmxlib.write_pmx(output_filename_pmx, this_pmx_obj, moreinfo=moreinfo)
# =========================================================================================================
# =========================================================================================================
# =========================================================================================================
# NOW PRINT MY RENAMINGS and other findings
filerecord_with_savings = zip(image_filerecords, mem_saved)
changed_files = [u for u in filerecord_with_savings if u[0].newname is not None]
core.MY_PRINT_FUNC("="*60)
if pil_cannot_inspect:
core.MY_PRINT_FUNC("WARNING: failed to inspect %d image files, these must be handled manually" % pil_cannot_inspect)
core.MY_PRINT_FUNC(pil_cannot_inspect_list)
if num_recompressed:
core.MY_PRINT_FUNC("Recompressed %d images! %s of disk space has been freed" % (num_recompressed, core.prettyprint_file_size(sum(mem_saved))))
if pil_imgext_mismatch:
core.MY_PRINT_FUNC("Renamed %d images that had incorrect extensions (included below)" % pil_imgext_mismatch)
oldname_list = [p[0].name for p in changed_files]
oldname_list_j = core.MY_JUSTIFY_STRINGLIST(oldname_list)
newname_list = [p[0].newname for p in changed_files]
newname_list_j = core.MY_JUSTIFY_STRINGLIST(newname_list)
savings_list = [("" if p[1]==0 else "saved " + core.prettyprint_file_size(p[1])) for p in changed_files]
zipped = list(zip(oldname_list_j, newname_list_j, savings_list))
zipped_and_sorted = sorted(zipped, key=lambda y: file_sort_textures.sortbydirdepth(y[0]))
for o,n,s in zipped_and_sorted:
# print 'from' with the case/separator it uses in the PMX
core.MY_PRINT_FUNC(" {:s} --> {:s} | {:s}".format(o, n, s))
core.MY_PRINT_FUNC("Done!")
return None
def main(moreinfo=False):
core.MY_PRINT_FUNC("Please enter name of PMX model file:")
input_filename_pmx = core.MY_FILEPROMPT_FUNC(".pmx")
# step zero: set up the translator thingy
translate_to_english.init_googletrans()
# texture sorting plan:
# 1. get startpath = basepath of input PMX
# 2. get lists of relevant files
# 2a. extract top-level 'neighbor' pmx files from all-set
# 3. ask about modifying neighbor PMX
# 4. read PMX: either target or target+all neighbor
# 5. "categorize files & normalize usages within PMX", NEW FUNC!!!
# 6. translate all names via Google Trans, don't even bother with local dict
# 7. mask out invalid windows filepath chars just to be safe
# 8. print proposed names & other findings
# for unused files under a folder, combine & replace with ***
# 9. ask for confirmation
# 10. zip backup (NEW FUNC!)
# 11. apply renaming, NEW FUNC! rename all including old PMXes on disk
# 12. get new names for PMXes, write PMX from mem to disk if any of its contents changed
# i.e. of all FileRecord with a new name, create a set of all the PMX that use them
# absolute path to directory holding the pmx
input_filename_pmx_abs = os.path.normpath(
os.path.abspath(input_filename_pmx))
startpath, input_filename_pmx_rel = os.path.split(input_filename_pmx_abs)
# =========================================================================================================
# =========================================================================================================
# =========================================================================================================
# first, build the list of ALL files that actually exist, then filter it down to neighbor PMXs and relevant files
relative_all_exist_files = file_sort_textures.walk_filetree_from_root(
startpath)
core.MY_PRINT_FUNC("ALL EXISTING FILES:", len(relative_all_exist_files))
# now fill "neighbor_pmx" by finding files without path separator that end in PMX
# these are relative paths tho
neighbor_pmx = [
f for f in relative_all_exist_files
if (f.lower().endswith(".pmx")) and (
os.path.sep not in f) and f != input_filename_pmx_rel
]
# no filtering, all files are relevant
relevant_exist_files = relative_all_exist_files
core.MY_PRINT_FUNC("NEIGHBOR PMX FILES:", len(neighbor_pmx))
# =========================================================================================================
# =========================================================================================================
# =========================================================================================================
# now ask if I care about the neighbors and read the PMXes into memory
pmx_filenames = [input_filename_pmx_rel]
if neighbor_pmx:
core.MY_PRINT_FUNC("")
info = [
"Detected %d top-level neighboring PMX files, these probably share the same filebase as the target."
% len(neighbor_pmx),
"If files are moved/renamed but the neighbors are not processed, the neighbor texture references will probably break.",
"Do you want to process all neighbors in addition to the target? (highly recommended)",
"1 = Yes, 2 = No"
]
r = core.MY_SIMPLECHOICE_FUNC((1, 2), info)
if r == 1:
core.MY_PRINT_FUNC("Processing target + all neighbor files")
# append neighbor PMX files onto the list of files to be processed
pmx_filenames += neighbor_pmx
else:
core.MY_PRINT_FUNC(
"WARNING: Processing only target, ignoring %d neighbor PMX files"
% len(neighbor_pmx))
# now read all the PMX objects & store in dict alongside the relative name
# dictionary where keys are filename and values are resulting pmx objects
all_pmx_obj = {}
for this_pmx_name in pmx_filenames:
this_pmx_obj = pmxlib.read_pmx(os.path.join(startpath, this_pmx_name),
moreinfo=moreinfo)
all_pmx_obj[this_pmx_name] = this_pmx_obj
# =========================================================================================================
# =========================================================================================================
# =========================================================================================================
# for each pmx, for each file on disk, match against files used in textures (case-insensitive) and replace with canonical name-on-disk
# also fill out how much and how each file is used, and unify dupes between files, all that good stuff
filerecord_list = file_sort_textures.categorize_files(
all_pmx_obj, relevant_exist_files, moreinfo)
# =========================================================================================================
# =========================================================================================================
# =========================================================================================================
# DETERMINE NEW NAMES FOR FILES
# how to remap: build a list of all destinations (lowercase) to see if any proposed change would lead to collision
all_new_names = set()
# get new names via google
# force it to use chunk-wise translate
newname_list = translate_to_english.google_translate(
[p.name for p in filerecord_list], strategy=1)
# now repair any windows-forbidden symbols that might have shown up after translation
newname_list = [
n.translate(invalid_windows_chars_ord) for n in newname_list
]
# iterate over the results in parallel with the FileRecord items
for p, newname in zip(filerecord_list, newname_list):
if newname != p.name:
# resolve potential collisions by adding numbers suffix to file names
# first need to make path absolute so get_unused_file_name can check the disk.
# then check uniqueness against files on disk and files in namelist (files that WILL be on disk)
newname = core.get_unused_file_name(os.path.join(
startpath, newname),
namelist=all_new_names)
# now dest path is guaranteed unique against other existing files & other proposed name changes
all_new_names.add(newname.lower())
# make the path no longer absolute: undo adding "startpath" above
newname = os.path.relpath(newname, startpath)
p.newname = newname
# =========================================================================================================
# =========================================================================================================
# =========================================================================================================
# NOW PRINT MY PROPOSED RENAMINGS and other findings
# isolate the ones with proposed renaming
translated_file = [u for u in filerecord_list if u.newname is not None]
if translated_file:
core.MY_PRINT_FUNC("=" * 60)
core.MY_PRINT_FUNC("Found %d JP filenames to be translated:" %
len(translated_file))
oldname_list = core.MY_JUSTIFY_STRINGLIST(
[p.name for p in translated_file])
newname_list = [p.newname for p in translated_file]
zipped = list(zip(oldname_list, newname_list))
zipped_and_sorted = sorted(
zipped, key=lambda y: file_sort_textures.sortbydirdepth(y[0]))
for o, n in zipped_and_sorted:
# print 'from' with the case/separator it uses in the PMX
core.MY_PRINT_FUNC(" {:s} --> {:s}".format(o, n))
core.MY_PRINT_FUNC("=" * 60)
else:
core.MY_PRINT_FUNC("No proposed file changes")
core.MY_PRINT_FUNC("Aborting: no files were changed")
return None
info = [
"Do you accept these new names/locations?", "1 = Yes, 2 = No (abort)"
]
r = core.MY_SIMPLECHOICE_FUNC((1, 2), info)
if r == 2:
core.MY_PRINT_FUNC("Aborting: no files were changed")
return None
# =========================================================================================================
# =========================================================================================================
# =========================================================================================================
# finally, do the actual renaming:
# first, create a backup of the folder
if MAKE_BACKUP_BEFORE_RENAMES:
r = file_sort_textures.make_zipfile_backup(startpath, BACKUP_SUFFIX)
if not r:
# this happens if the backup failed somehow AND the user decided to quit
core.MY_PRINT_FUNC("Aborting: no files were changed")
return None
# do all renaming on disk and in PMXes, and also handle the print statements
file_sort_textures.apply_file_renaming(all_pmx_obj, filerecord_list,
startpath)
# write out
for this_pmx_name, this_pmx_obj in all_pmx_obj.items():
# what name do i write this pmx to? it may be different now! find it in the FileRecord!
# this script does not filter filerecord_list so it is guaranteed to hae a record
rec = None
for r in filerecord_list:
if r.name == this_pmx_name:
rec = r
break
if rec.newname is None:
# if there is no new name, write back to the name it had previously
new_pmx_name = rec.name
else:
# if there is a new name, write to the new name
new_pmx_name = rec.newname
# make the name absolute
output_filename_pmx = os.path.join(startpath, new_pmx_name)
# write it, overwriting the existing file at that name
pmxlib.write_pmx(output_filename_pmx, this_pmx_obj, moreinfo=moreinfo)
core.MY_PRINT_FUNC("Done!")
return None