def __init__(self, data=None, offset=0, filename=None):
self.data = None
self.__gim_files = []
self.gimconv = GimConverter()
if not data == None:
self.load_data(data, offset)
elif not filename == None:
self.load_file(filename)
def __init__(self, data = None, offset = 0, filename = None):
self.data = None
self.__gim_files = []
self.gimconv = GimConverter()
if not data == None:
self.load_data(data, offset)
elif not filename == None:
self.load_file(filename)
import shutil
import tempfile
import threading
import time
# from bitstring import ConstBitStream
import common
from backup import backup_files
from dupe_db import DupesDB
from list_files import list_all_files
from gim_converter import GimConverter, QuantizeType
from model_pak import ModelPak
_CONV = GimConverter()
_DUPE_DB = DupesDB()
SKIP_CONV = [
"save_icon0.png", "save_icon0_t.png", "save_new_icon0.png", "save_pic1.png"
]
FORCE_QUANTIZE = [
(re.compile(ur"art_chip_002_\d\d\d.*", re.UNICODE), QuantizeType.index8),
(re.compile(ur"bgd_\d\d\d.*", re.UNICODE), QuantizeType.index8),
(re.compile(ur"bustup_\d\d_\d\d.*", re.UNICODE), QuantizeType.index8),
(re.compile(ur"(cutin|gallery|kotodama|present)_icn_\d\d\d.*",
re.UNICODE), QuantizeType.index8),
]
MIN_INTERVAL = 0.100
def copy_gfx(self):
gfx_dir = os.path.join(self.editor_data_dir, "gfx")
if os.path.isdir(gfx_dir):
shutil.rmtree(gfx_dir)
os.makedirs(gfx_dir)
progress = QProgressDialog("", "Abort", 0, 0, self)
progress.setWindowTitle("Copying GFX...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setMinimumDuration(0)
progress.setValue(0)
progress.setAutoClose(False)
progress.setLabelText("Setting up GFX dir.")
progress.setMaximum(5)
progress.setValue(0)
# Extract the images we can't just take directly from the game's data.
gfx_bin = zipfile.ZipFile("data/gfx_base.bin", "r")
progress.setValue(1)
gfx_enc = gfx_bin.open("gfx_base.bin")
progress.setValue(2)
gfx_dec = cStringIO.StringIO()
base64.decode(gfx_enc, gfx_dec)
progress.setValue(3)
gfx_base = zipfile.ZipFile(gfx_dec, "r")
progress.setValue(4)
gfx_base.extractall(gfx_dir)
progress.setValue(5)
gfx_base.close()
gfx_dec.close()
gfx_enc.close()
gfx_bin.close()
# We can mostly loop this.
gfx_data = [
("ammo", "kotodama_icn_???.gim"),
("bgd", "bgd_???.gim"),
("cutin", "cutin_icn_???.gim"),
("events", "gallery_icn_???.gim"),
("movies", "bin_movie_gallery_l.pak/0000/000[1789].gim"),
("movies", "bin_movie_gallery_l.pak/0000/00[123]?.gim"),
("nametags", "tex_system.pak/00[12]?.gim"),
("nametags", "tex_system.pak/003[0123456].gim"),
("presents", "present_icn_???.gim"),
("sprites", "bustup_??_??.gim"),
("sprites", "stand_??_??.gmo"),
]
for (dir, file_glob) in gfx_data:
out_dir = os.path.join(gfx_dir, dir)
files = glob.glob(os.path.join(self.umdimage_dir, file_glob))
progress.setLabelText("Copying %s." % dir)
progress.setMaximum(len(files))
progress.setValue(0)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
for i, image in enumerate(files):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
src = image
dest = os.path.join(out_dir, os.path.basename(src))
shutil.copy(src, dest)
progress.setValue(len(files))
progress.setLabelText("Copying font.")
progress.setMaximum(4)
progress.setValue(0)
# The font we have to get from umdimage2.
font_dir = os.path.join(gfx_dir, "font")
if not os.path.isdir(font_dir):
os.makedirs(font_dir)
progress.setValue(1)
# And convert to PNG with an alpha channel so our editor can use it.
font1 = font_bmp_to_alpha(os.path.join(self.umdimage2_dir, "font.pak", "0000.bmp"))
progress.setValue(2)
font2 = font_bmp_to_alpha(os.path.join(self.umdimage2_dir, "font.pak", "0002.bmp"))
progress.setValue(3)
font1.save(os.path.join(font_dir, "Font01.png"))
font2.save(os.path.join(font_dir, "Font02.png"))
shutil.copy(os.path.join(self.umdimage2_dir, "font.pak", "0001.font"), os.path.join(font_dir, "Font01.font"))
shutil.copy(os.path.join(self.umdimage2_dir, "font.pak", "0003.font"), os.path.join(font_dir, "Font02.font"))
progress.setValue(4)
# And then the flash files. This'll be fun.
flash_dir = os.path.join(gfx_dir, "flash")
if not os.path.isdir(flash_dir):
os.makedirs(flash_dir)
# Because there's so many in so many different places, I just stored a list
# of the flash files we need in the gfx_base archive. So let's load that.
with open(os.path.join(gfx_dir, "fla.txt"), "rb") as fla:
fla_list = fla.readlines()
progress.setLabelText("Copying flash.")
progress.setMaximum(len(fla_list))
progress.setValue(0)
for i, flash in enumerate(fla_list):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
flash = flash.strip()
fla_name = flash[:7] # fla_###
src = os.path.join(self.umdimage_dir, flash)
dest = os.path.join(flash_dir, "%s.gim" % fla_name)
shutil.copy(src, dest)
progress.setValue(len(fla_list))
# We have a couple sets of files that aren't named the way we want them to
# be, just because of how they're stored in umdimage.
progress.setLabelText("Renaming files.")
to_rename = [
("movies", "movie_%03d.gim", range(32)),
("nametags", "%02d.gim", range(23) + [24, 25, 30, 31]),
]
for (folder, pattern, nums) in to_rename:
folder = os.path.join(gfx_dir, folder)
files = glob.glob(os.path.join(folder, "*.gim"))
progress.setMaximum(len(files))
progress.setValue(0)
for i, image in enumerate(files):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
src = image
dest = os.path.join(folder, pattern % nums[i])
if os.path.isfile(dest):
os.remove(dest)
shutil.move(src, dest)
sprite_dir = os.path.join(gfx_dir, "sprites")
gmo_files = glob.glob(os.path.join(sprite_dir, "*.gmo"))
progress.setLabelText("Extracting GMO files.")
progress.setValue(0)
progress.setMaximum(len(gmo_files))
for i, gmo_file in enumerate(gmo_files):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
name, ext = os.path.splitext(os.path.basename(gmo_file))
gim_file = os.path.join(sprite_dir, name + ".gim")
gmo = GmoFile(filename = gmo_file)
# Once we've loaded it, we're all done with it, so make it go away.
os.remove(gmo_file)
if gmo.gim_count() == 0:
continue
gim = gmo.get_gim(0)
with open(gim_file, "wb") as f:
gim.tofile(f)
if self.ui.chkGimToPng.isChecked():
gim_files = glob.glob(os.path.join(gfx_dir, "*", "*.gim"))
progress.setLabelText("Converting GIM to PNG.")
progress.setValue(0)
progress.setMaximum(len(gim_files))
converter = GimConverter()
for i, gim_file in enumerate(gim_files):
progress.setValue(i)
if progress.wasCanceled():
return
converter.gim_to_png(gim_file)
os.remove(gim_file)
progress.close()
self.gfx_dir = gfx_dir
self.ui.grpStep5.setEnabled(False)
self.ui.grpStep6.setEnabled(True)
class GmoFile():
def __init__(self, data = None, offset = 0, filename = None):
self.data = None
self.__gim_files = []
self.gimconv = GimConverter()
if not data == None:
self.load_data(data, offset)
elif not filename == None:
self.load_file(filename)
def load_file(self, filename):
data = BitStream(filename = filename)
self.load_data(data)
def load_data(self, data, offset = 0):
if not data[offset * 8 : offset * 8 + GMO_MAGIC.len] == GMO_MAGIC:
_LOGGER.error("GMO header not found at 0x%04X." % offset)
return
data.bytepos = offset + GMO_SIZE_OFFSET
gmo_size = data.read("uintle:32") + GMO_SIZE_DIFF
self.data = BitStream(data[offset * 8 : (offset + gmo_size) * 8])
self.__find_gims()
def save(self, filename):
with open(filename, "wb") as f:
self.data.tofile(f)
def __find_gims(self):
if self.data == None:
return
self.__gim_files = []
for gim_start in self.data.findall(GIM_MAGIC, bytealigned = True):
gim_size_pos = gim_start + (GIM_SIZE_OFFSET * 8) # Bit pos.
gim_size = self.data[gim_size_pos : gim_size_pos + 32].uintle + GIM_SIZE_DIFF
# And turn it into a byte position.
gim_start /= 8
self.__gim_files.append((gim_start, gim_size))
def gim_count(self):
return len(self.__gim_files)
def get_gim(self, gim_id):
if gim_id >= self.gim_count():
raise GimIndexError("Invalid GIM ID.")
gim_start, gim_size = self.__gim_files[gim_id]
gim_data = self.data[gim_start * 8 : (gim_start + gim_size) * 8]
return gim_data
def replace_png_file(self, gim_id, filename, quantize_to_fit = True):
if quantize_to_fit:
quantize_order = [QuantizeType.auto, QuantizeType.index8, QuantizeType.index4]
else:
quantize_order = [QuantizeType.auto]
quantize_id = 0
(fd, temp_gim) = tempfile.mkstemp(suffix = ".gim", prefix = "sdse-")
os.close(fd) # Don't need the open file handle.
while True:
self.gimconv.png_to_gim(filename, temp_gim, quantize_order[quantize_id])
try:
self.replace_gim_file(gim_id, temp_gim)
except GimSizeError:
quantize_id += 1
except GimIndexError:
os.remove(temp_gim)
raise
else:
# If we didn't except, that means we succeeded, so we can leave.
_LOGGER.debug("Quantized PNG to %s" % quantize_order[quantize_id])
break
if quantize_id > len(quantize_order):
_LOGGER.error("Unable to convert %s into a GIM small enough to insert." % filename)
break
os.remove(temp_gim)
def replace_gim_file(self, gim_id, filename):
gim_data = BitStream(filename = filename)
self.replace_gim(gim_id, gim_data)
def replace_gim(self, gim_id, gim_data):
if gim_id >= self.gim_count():
raise GimIndexError("Invalid GIM ID.")
gim_start, gim_size = self.__gim_files[gim_id]
if gim_data.len / 8 > gim_size:
raise GimSizeError("GIM too large. %d bytes > %d bytes" % (gim_data.len / 8, gim_size))
# return
self.data.overwrite(gim_data, gim_start * 8)
# Leave the length alone, though, because we know we have that much space
# to work with from the original GIM file that was there, and there's no
# point in shrinking that down if someone happens to want to re-replace
# this GIM file without reloading the whole thing.
def extract(self, directory, to_png = False):
if not os.path.isdir(directory):
os.makedirs(directory)
for id in range(self.gim_count()):
gim = self.get_gim(id)
out_gim = os.path.join(directory, "%04d.gim" % id)
out_png = os.path.join(directory, "%04d.png" % id)
with open(out_gim, "wb") as f:
gim.tofile(f)
if to_png:
self.gimconv.gim_to_png(out_gim, out_png)
os.remove(out_gim)
def copy_gfx(self):
gfx_dir = os.path.join(self.editor_data_dir, "gfx")
if os.path.isdir(gfx_dir):
shutil.rmtree(gfx_dir)
os.makedirs(gfx_dir)
progress = QProgressDialog("", "Abort", 0, 0, self)
progress.setWindowTitle("Copying GFX...")
progress.setWindowModality(Qt.Qt.WindowModal)
progress.setMinimumDuration(0)
progress.setValue(0)
progress.setAutoClose(False)
progress.setLabelText("Setting up GFX dir.")
progress.setMaximum(5)
progress.setValue(0)
# Extract the images we can't just take directly from the game's data.
gfx_bin = zipfile.ZipFile("data/gfx_base.zip", "r")
progress.setValue(1)
progress.setValue(2)
gfx_bin.extractall(gfx_dir)
progress.setValue(5)
gfx_bin.close()
# We can mostly loop this.
gfx_data = [
("ammo", "kotodama_icn_???.gim"),
("bgd", "bgd_???.gim"),
("cutin", "cutin_icn_???.gim"),
("events", "gallery_icn_???.gim"),
("movies", "bin_movie_gallery_l.pak/0000/000[1789].gim"),
("movies", "bin_movie_gallery_l.pak/0000/00[123]?.gim"),
("movies", "gallery_ico_m_none.gim"),
("movies", "gallery_thumbnail_m_???.gim"),
("nametags", "tex_system.pak/00[12]?.gim"),
("nametags", "tex_system.pak/003[0123456].gim"),
("presents", "present_icn_???.gim"),
("sprites", "bustup_??_??.gim"),
("sprites", "stand_??_??.gmo"),
]
for (dir, file_glob) in gfx_data:
out_dir = os.path.join(gfx_dir, dir)
files = glob.glob(os.path.join(self.data0_dir, file_glob))
progress.setLabelText("Copying %s." % dir)
progress.setMaximum(len(files))
progress.setValue(0)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
for i, image in enumerate(files):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
src = image
dest = os.path.join(out_dir, os.path.basename(src))
shutil.copy(src, dest)
progress.setValue(len(files))
progress.setLabelText("Copying font.")
progress.setMaximum(4)
progress.setValue(0)
# The font we have to get from umdimage2.
font_dir = os.path.join(gfx_dir, "font")
if not os.path.isdir(font_dir):
os.makedirs(font_dir)
progress.setValue(1)
# And convert to PNG with an alpha channel so our editor can use it.
font1 = font_bmp_to_alpha(os.path.join(self.data01_dir, "jp", "font", "font.pak", "0000.bmp"))
progress.setValue(2)
font2 = font_bmp_to_alpha(os.path.join(self.data01_dir, "jp", "font", "font.pak", "0002.bmp"))
progress.setValue(3)
font1.save(os.path.join(font_dir, "Font01.png"))
font2.save(os.path.join(font_dir, "Font02.png"))
shutil.copy(
os.path.join(self.data01_dir, "jp", "font", "font.pak", "0001.font"), os.path.join(font_dir, "Font01.font")
)
shutil.copy(
os.path.join(self.data01_dir, "jp", "font", "font.pak", "0003.font"), os.path.join(font_dir, "Font02.font")
)
progress.setValue(4)
# And then the flash files. This'll be fun.
flash_dir = os.path.join(gfx_dir, "flash")
if not os.path.isdir(flash_dir):
os.makedirs(flash_dir)
# Because there's so many in so many different places, I just stored a list
# of the flash files we need in the gfx_base archive. So let's load that.
with open(os.path.join(gfx_dir, "fla.txt"), "rb") as fla:
fla_list = fla.readlines()
progress.setLabelText("Copying flash.")
progress.setMaximum(len(fla_list))
progress.setValue(0)
for i, flash in enumerate(fla_list):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
flash = flash.strip()
fla_name = flash[:7] # fla_###
src = os.path.join(self.data01_dir, "all", "flash", flash)
dest = os.path.join(flash_dir, "%s.gim" % fla_name)
shutil.copy(src, dest)
progress.setValue(len(fla_list))
# We have a couple sets of files that aren't named the way we want them to
# be, just because of how they're stored in umdimage.
progress.setLabelText("Renaming files.")
to_rename = [("movies", "movie_%03d.gim", range(32)), ("nametags", "%02d.gim", range(23) + [24, 25, 30, 31])]
for (folder, pattern, nums) in to_rename:
folder = os.path.join(gfx_dir, folder)
files = glob.glob(os.path.join(folder, "*.gim"))
progress.setMaximum(len(files))
progress.setValue(0)
for i, image in enumerate(files):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
src = image
dest = os.path.join(folder, pattern % nums[i])
if os.path.isfile(dest):
os.remove(dest)
shutil.move(src, dest)
sprite_dir = os.path.join(gfx_dir, "sprites")
gmo_files = glob.glob(os.path.join(sprite_dir, "*.gmo"))
progress.setLabelText("Extracting GMO files.")
progress.setValue(0)
progress.setMaximum(len(gmo_files))
for i, gmo_file in enumerate(gmo_files):
if i % 10 == 0:
progress.setValue(i)
if progress.wasCanceled():
return
name, ext = os.path.splitext(os.path.basename(gmo_file))
gim_file = os.path.join(sprite_dir, name + ".gim")
gmo = GmoFile(filename=gmo_file)
# Once we've loaded it, we're all done with it, so make it go away.
os.remove(gmo_file)
if gmo.gim_count() == 0:
continue
gim = gmo.get_gim(0)
with open(gim_file, "wb") as f:
gim.tofile(f)
if self.ui.chkGimToPng.isChecked():
gim_files = glob.glob(os.path.join(gfx_dir, "*", "*.gim"))
progress.setLabelText("Converting GIM to PNG.")
progress.setValue(0)
progress.setMaximum(len(gim_files))
converter = GimConverter()
for i, gim_file in enumerate(gim_files):
progress.setValue(i)
if progress.wasCanceled():
return
converter.gim_to_png(gim_file)
os.remove(gim_file)
progress.close()
self.gfx_dir = gfx_dir
self.ui.grpStep5.setEnabled(False)
self.ui.grpStep6.setEnabled(True)
class GmoFile():
def __init__(self, data=None, offset=0, filename=None):
self.data = None
self.__gim_files = []
self.gimconv = GimConverter()
if not data == None:
self.load_data(data, offset)
elif not filename == None:
self.load_file(filename)
def load_file(self, filename):
data = BitStream(filename=filename)
self.load_data(data)
def load_data(self, data, offset=0):
if not data[offset * 8:offset * 8 + GMO_MAGIC.len] == GMO_MAGIC:
_LOGGER.error("GMO header not found at 0x%04X." % offset)
return
data.bytepos = offset + GMO_SIZE_OFFSET
gmo_size = data.read("uintle:32") + GMO_SIZE_DIFF
self.data = BitStream(data[offset * 8:(offset + gmo_size) * 8])
self.__find_gims()
def save(self, filename):
with open(filename, "wb") as f:
self.data.tofile(f)
def __find_gims(self):
if self.data == None:
return
self.__gim_files = []
for gim_start in self.data.findall(GIM_MAGIC, bytealigned=True):
gim_size_pos = gim_start + (GIM_SIZE_OFFSET * 8) # Bit pos.
gim_size = self.data[gim_size_pos:gim_size_pos +
32].uintle + GIM_SIZE_DIFF
# And turn it into a byte position.
gim_start /= 8
self.__gim_files.append((gim_start, gim_size))
def gim_count(self):
return len(self.__gim_files)
def get_gim(self, gim_id):
if gim_id >= self.gim_count():
raise GimIndexError("Invalid GIM ID.")
gim_start, gim_size = self.__gim_files[gim_id]
gim_data = self.data[gim_start * 8:(gim_start + gim_size) * 8]
return gim_data
def replace_png_file(self, gim_id, filename, quantize_to_fit=True):
if quantize_to_fit:
quantize_order = [
QuantizeType.auto, QuantizeType.index8, QuantizeType.index4
]
else:
quantize_order = [QuantizeType.auto]
quantize_id = 0
(fd, temp_gim) = tempfile.mkstemp(suffix=".gim", prefix="sdse-")
os.close(fd) # Don't need the open file handle.
while True:
self.gimconv.png_to_gim(filename, temp_gim,
quantize_order[quantize_id])
try:
self.replace_gim_file(gim_id, temp_gim)
except GimSizeError:
quantize_id += 1
except GimIndexError:
os.remove(temp_gim)
raise
else:
# If we didn't except, that means we succeeded, so we can leave.
_LOGGER.debug("Quantized PNG to %s" %
quantize_order[quantize_id])
break
if quantize_id > len(quantize_order):
_LOGGER.error(
"Unable to convert %s into a GIM small enough to insert." %
filename)
break
os.remove(temp_gim)
def replace_gim_file(self, gim_id, filename):
gim_data = BitStream(filename=filename)
self.replace_gim(gim_id, gim_data)
def replace_gim(self, gim_id, gim_data):
if gim_id >= self.gim_count():
raise GimIndexError("Invalid GIM ID.")
gim_start, gim_size = self.__gim_files[gim_id]
if gim_data.len / 8 > gim_size:
raise GimSizeError("GIM too large. %d bytes > %d bytes" %
(gim_data.len / 8, gim_size))
# return
self.data.overwrite(gim_data, gim_start * 8)
# Leave the length alone, though, because we know we have that much space
# to work with from the original GIM file that was there, and there's no
# point in shrinking that down if someone happens to want to re-replace
# this GIM file without reloading the whole thing.
def extract(self, directory, to_png=False):
if not os.path.isdir(directory):
os.makedirs(directory)
for id in range(self.gim_count()):
gim = self.get_gim(id)
out_gim = os.path.join(directory, "%04d.gim" % id)
out_png = os.path.join(directory, "%04d.png" % id)
with open(out_gim, "wb") as f:
gim.tofile(f)
if to_png:
self.gimconv.gim_to_png(out_gim, out_png)
os.remove(out_gim)
