Beispiel #1
0
    def __init__(self, data=None, offset=0, filename=None):
        self.data = None
        self.__gim_files = []

        self.gimconv = GimConverter()

        if not data == None:
            self.load_data(data, offset)
        elif not filename == None:
            self.load_file(filename)
 def __init__(self, data = None, offset = 0, filename = None):
   self.data = None
   self.__gim_files = []
   
   self.gimconv = GimConverter()
   
   if not data == None:
     self.load_data(data, offset)
   elif not filename == None:
     self.load_file(filename)
import shutil
import tempfile
import threading
import time

# from bitstring import ConstBitStream

import common

from backup import backup_files
from dupe_db import DupesDB
from list_files import list_all_files
from gim_converter import GimConverter, QuantizeType
from model_pak import ModelPak

_CONV = GimConverter()
_DUPE_DB = DupesDB()

SKIP_CONV = [
    "save_icon0.png", "save_icon0_t.png", "save_new_icon0.png", "save_pic1.png"
]

FORCE_QUANTIZE = [
    (re.compile(ur"art_chip_002_\d\d\d.*", re.UNICODE), QuantizeType.index8),
    (re.compile(ur"bgd_\d\d\d.*", re.UNICODE), QuantizeType.index8),
    (re.compile(ur"bustup_\d\d_\d\d.*", re.UNICODE), QuantizeType.index8),
    (re.compile(ur"(cutin|gallery|kotodama|present)_icn_\d\d\d.*",
                re.UNICODE), QuantizeType.index8),
]

MIN_INTERVAL = 0.100
 def copy_gfx(self):
   gfx_dir = os.path.join(self.editor_data_dir, "gfx")
   
   if os.path.isdir(gfx_dir):
     shutil.rmtree(gfx_dir)
   
   os.makedirs(gfx_dir)
   
   progress = QProgressDialog("", "Abort", 0, 0, self)
   progress.setWindowTitle("Copying GFX...")
   progress.setWindowModality(Qt.Qt.WindowModal)
   progress.setMinimumDuration(0)
   progress.setValue(0)
   progress.setAutoClose(False)
   
   progress.setLabelText("Setting up GFX dir.")
   progress.setMaximum(5)
   progress.setValue(0)
   
   # Extract the images we can't just take directly from the game's data.
   gfx_bin = zipfile.ZipFile("data/gfx_base.bin", "r")
   progress.setValue(1)
   gfx_enc = gfx_bin.open("gfx_base.bin")
   progress.setValue(2)
   gfx_dec = cStringIO.StringIO()
   base64.decode(gfx_enc, gfx_dec)
   progress.setValue(3)
   gfx_base = zipfile.ZipFile(gfx_dec, "r")
   progress.setValue(4)
   gfx_base.extractall(gfx_dir)
   progress.setValue(5)
   gfx_base.close()
   gfx_dec.close()
   gfx_enc.close()
   gfx_bin.close()
   
   # We can mostly loop this.
   gfx_data = [
     ("ammo",      "kotodama_icn_???.gim"),
     ("bgd",       "bgd_???.gim"),
     ("cutin",     "cutin_icn_???.gim"),
     ("events",    "gallery_icn_???.gim"),
     ("movies",    "bin_movie_gallery_l.pak/0000/000[1789].gim"),
     ("movies",    "bin_movie_gallery_l.pak/0000/00[123]?.gim"),
     ("nametags",  "tex_system.pak/00[12]?.gim"),
     ("nametags",  "tex_system.pak/003[0123456].gim"),
     ("presents",  "present_icn_???.gim"),
     ("sprites",   "bustup_??_??.gim"),
     ("sprites",   "stand_??_??.gmo"),
   ]
   
   for (dir, file_glob) in gfx_data:
     out_dir = os.path.join(gfx_dir, dir)
     files   = glob.glob(os.path.join(self.umdimage_dir, file_glob))
     
     progress.setLabelText("Copying %s." % dir)
     progress.setMaximum(len(files))
     progress.setValue(0)
     
     if not os.path.isdir(out_dir):
       os.makedirs(out_dir)
     
     for i, image in enumerate(files):
       if i % 10 == 0:
         progress.setValue(i)
       
       if progress.wasCanceled():
         return
       
       src  = image
       dest = os.path.join(out_dir, os.path.basename(src))
       shutil.copy(src, dest)
     
     progress.setValue(len(files))
   
   progress.setLabelText("Copying font.")
   progress.setMaximum(4)
   progress.setValue(0)
   
   # The font we have to get from umdimage2.
   font_dir = os.path.join(gfx_dir, "font")
   if not os.path.isdir(font_dir):
     os.makedirs(font_dir)
   
   progress.setValue(1)
   # And convert to PNG with an alpha channel so our editor can use it.
   font1 = font_bmp_to_alpha(os.path.join(self.umdimage2_dir, "font.pak", "0000.bmp"))
   progress.setValue(2)
   font2 = font_bmp_to_alpha(os.path.join(self.umdimage2_dir, "font.pak", "0002.bmp"))
   progress.setValue(3)
   
   font1.save(os.path.join(font_dir, "Font01.png"))
   font2.save(os.path.join(font_dir, "Font02.png"))
   shutil.copy(os.path.join(self.umdimage2_dir, "font.pak", "0001.font"), os.path.join(font_dir, "Font01.font"))
   shutil.copy(os.path.join(self.umdimage2_dir, "font.pak", "0003.font"), os.path.join(font_dir, "Font02.font"))
   
   progress.setValue(4)
   
   # And then the flash files. This'll be fun.
   flash_dir = os.path.join(gfx_dir, "flash")
   if not os.path.isdir(flash_dir):
     os.makedirs(flash_dir)
   
   # Because there's so many in so many different places, I just stored a list
   # of the flash files we need in the gfx_base archive. So let's load that.
   with open(os.path.join(gfx_dir, "fla.txt"), "rb") as fla:
     fla_list = fla.readlines()
     
     progress.setLabelText("Copying flash.")
     progress.setMaximum(len(fla_list))
     progress.setValue(0)
     
     for i, flash in enumerate(fla_list):
       if i % 10 == 0:
         progress.setValue(i)
       
       if progress.wasCanceled():
         return
       
       flash = flash.strip()
       fla_name = flash[:7] # fla_###
       
       src  = os.path.join(self.umdimage_dir, flash)
       dest = os.path.join(flash_dir, "%s.gim" % fla_name)
       
       shutil.copy(src, dest)
       
     progress.setValue(len(fla_list))
   
   # We have a couple sets of files that aren't named the way we want them to
   # be, just because of how they're stored in umdimage.
   progress.setLabelText("Renaming files.")
   to_rename = [
     ("movies",    "movie_%03d.gim", range(32)),
     ("nametags",  "%02d.gim", range(23) + [24, 25, 30, 31]),
   ]
   
   for (folder, pattern, nums) in to_rename:
     folder  = os.path.join(gfx_dir, folder)
     files   = glob.glob(os.path.join(folder, "*.gim"))
     
     progress.setMaximum(len(files))
     progress.setValue(0)
     
     for i, image in enumerate(files):
       if i % 10 == 0:
         progress.setValue(i)
       
       if progress.wasCanceled():
         return
       
       src  = image
       dest = os.path.join(folder, pattern % nums[i])
       
       if os.path.isfile(dest):
         os.remove(dest)
       
       shutil.move(src, dest)
   
   sprite_dir = os.path.join(gfx_dir, "sprites")
   gmo_files = glob.glob(os.path.join(sprite_dir, "*.gmo"))
   
   progress.setLabelText("Extracting GMO files.")
   progress.setValue(0)
   progress.setMaximum(len(gmo_files))
   
   for i, gmo_file in enumerate(gmo_files):
     if i % 10 == 0:
       progress.setValue(i)
     
     if progress.wasCanceled():
       return
     
     name, ext = os.path.splitext(os.path.basename(gmo_file))
     gim_file  = os.path.join(sprite_dir, name + ".gim")
     
     gmo = GmoFile(filename = gmo_file)
     
     # Once we've loaded it, we're all done with it, so make it go away.
     os.remove(gmo_file)
     
     if gmo.gim_count() == 0:
       continue
     
     gim = gmo.get_gim(0)
     
     with open(gim_file, "wb") as f:
       gim.tofile(f)
   
   if self.ui.chkGimToPng.isChecked():
     gim_files = glob.glob(os.path.join(gfx_dir, "*", "*.gim"))
     
     progress.setLabelText("Converting GIM to PNG.")
     progress.setValue(0)
     progress.setMaximum(len(gim_files))
     
     converter = GimConverter()
     
     for i, gim_file in enumerate(gim_files):
       progress.setValue(i)
       if progress.wasCanceled():
         return
       
       converter.gim_to_png(gim_file)
       os.remove(gim_file)
   
   progress.close()
   
   self.gfx_dir = gfx_dir
   
   self.ui.grpStep5.setEnabled(False)
   self.ui.grpStep6.setEnabled(True)
class GmoFile():
  def __init__(self, data = None, offset = 0, filename = None):
    self.data = None
    self.__gim_files = []
    
    self.gimconv = GimConverter()
    
    if not data == None:
      self.load_data(data, offset)
    elif not filename == None:
      self.load_file(filename)
  
  def load_file(self, filename):
    data = BitStream(filename = filename)
    self.load_data(data)
  
  def load_data(self, data, offset = 0):
    if not data[offset * 8 : offset * 8 + GMO_MAGIC.len] == GMO_MAGIC:
      _LOGGER.error("GMO header not found at 0x%04X." % offset)
      return
    
    data.bytepos = offset + GMO_SIZE_OFFSET
    gmo_size = data.read("uintle:32") + GMO_SIZE_DIFF
    
    self.data = BitStream(data[offset * 8 : (offset + gmo_size) * 8])
    
    self.__find_gims()
  
  def save(self, filename):
    with open(filename, "wb") as f:
      self.data.tofile(f)
  
  def __find_gims(self):
    if self.data == None:
      return
    
    self.__gim_files = []
    
    for gim_start in self.data.findall(GIM_MAGIC, bytealigned = True):
      gim_size_pos  = gim_start + (GIM_SIZE_OFFSET * 8) # Bit pos.
      gim_size      = self.data[gim_size_pos : gim_size_pos + 32].uintle + GIM_SIZE_DIFF
      
      # And turn it into a byte position.
      gim_start /= 8
      self.__gim_files.append((gim_start, gim_size))
  
  def gim_count(self):
    return len(self.__gim_files)
  
  def get_gim(self, gim_id):
    if gim_id >= self.gim_count():
      raise GimIndexError("Invalid GIM ID.")
    
    gim_start, gim_size = self.__gim_files[gim_id]
    gim_data = self.data[gim_start * 8 : (gim_start + gim_size) * 8]
    
    return gim_data
  
  def replace_png_file(self, gim_id, filename, quantize_to_fit = True):
  
    if quantize_to_fit:
      quantize_order = [QuantizeType.auto, QuantizeType.index8, QuantizeType.index4]
    else:
      quantize_order = [QuantizeType.auto]
    quantize_id = 0
    
    (fd, temp_gim) = tempfile.mkstemp(suffix = ".gim", prefix = "sdse-")
    os.close(fd) # Don't need the open file handle.
    
    while True:
      self.gimconv.png_to_gim(filename, temp_gim, quantize_order[quantize_id])
      
      try:
        self.replace_gim_file(gim_id, temp_gim)
      except GimSizeError:
        quantize_id += 1
      except GimIndexError:
        os.remove(temp_gim)
        raise
      else:
        # If we didn't except, that means we succeeded, so we can leave.
        _LOGGER.debug("Quantized PNG to %s" % quantize_order[quantize_id])
        break
      
      if quantize_id > len(quantize_order):
        _LOGGER.error("Unable to convert %s into a GIM small enough to insert." % filename)
        break
    
    os.remove(temp_gim)
  
  def replace_gim_file(self, gim_id, filename):
    gim_data = BitStream(filename = filename)
    self.replace_gim(gim_id, gim_data)
  
  def replace_gim(self, gim_id, gim_data):
    if gim_id >= self.gim_count():
      raise GimIndexError("Invalid GIM ID.")
    
    gim_start, gim_size = self.__gim_files[gim_id]
    
    if gim_data.len / 8 > gim_size:
      raise GimSizeError("GIM too large. %d bytes > %d bytes" % (gim_data.len / 8, gim_size))
      # return
    
    self.data.overwrite(gim_data, gim_start * 8)
    
    # Leave the length alone, though, because we know we have that much space
    # to work with from the original GIM file that was there, and there's no
    # point in shrinking that down if someone happens to want to re-replace
    # this GIM file without reloading the whole thing.
  
  def extract(self, directory, to_png = False):
    if not os.path.isdir(directory):
      os.makedirs(directory)
    
    for id in range(self.gim_count()):
      gim = self.get_gim(id)
      
      out_gim = os.path.join(directory, "%04d.gim" % id)
      out_png = os.path.join(directory, "%04d.png" % id)
      
      with open(out_gim, "wb") as f:
        gim.tofile(f)
      
      if to_png:
        self.gimconv.gim_to_png(out_gim, out_png)
        os.remove(out_gim)
    def copy_gfx(self):
        gfx_dir = os.path.join(self.editor_data_dir, "gfx")

        if os.path.isdir(gfx_dir):
            shutil.rmtree(gfx_dir)

        os.makedirs(gfx_dir)

        progress = QProgressDialog("", "Abort", 0, 0, self)
        progress.setWindowTitle("Copying GFX...")
        progress.setWindowModality(Qt.Qt.WindowModal)
        progress.setMinimumDuration(0)
        progress.setValue(0)
        progress.setAutoClose(False)

        progress.setLabelText("Setting up GFX dir.")
        progress.setMaximum(5)
        progress.setValue(0)

        # Extract the images we can't just take directly from the game's data.
        gfx_bin = zipfile.ZipFile("data/gfx_base.zip", "r")
        progress.setValue(1)
        progress.setValue(2)
        gfx_bin.extractall(gfx_dir)
        progress.setValue(5)
        gfx_bin.close()

        # We can mostly loop this.
        gfx_data = [
            ("ammo", "kotodama_icn_???.gim"),
            ("bgd", "bgd_???.gim"),
            ("cutin", "cutin_icn_???.gim"),
            ("events", "gallery_icn_???.gim"),
            ("movies", "bin_movie_gallery_l.pak/0000/000[1789].gim"),
            ("movies", "bin_movie_gallery_l.pak/0000/00[123]?.gim"),
            ("movies", "gallery_ico_m_none.gim"),
            ("movies", "gallery_thumbnail_m_???.gim"),
            ("nametags", "tex_system.pak/00[12]?.gim"),
            ("nametags", "tex_system.pak/003[0123456].gim"),
            ("presents", "present_icn_???.gim"),
            ("sprites", "bustup_??_??.gim"),
            ("sprites", "stand_??_??.gmo"),
        ]

        for (dir, file_glob) in gfx_data:
            out_dir = os.path.join(gfx_dir, dir)
            files = glob.glob(os.path.join(self.data0_dir, file_glob))

            progress.setLabelText("Copying %s." % dir)
            progress.setMaximum(len(files))
            progress.setValue(0)

            if not os.path.isdir(out_dir):
                os.makedirs(out_dir)

            for i, image in enumerate(files):
                if i % 10 == 0:
                    progress.setValue(i)

                if progress.wasCanceled():
                    return

                src = image
                dest = os.path.join(out_dir, os.path.basename(src))
                shutil.copy(src, dest)

            progress.setValue(len(files))

        progress.setLabelText("Copying font.")
        progress.setMaximum(4)
        progress.setValue(0)

        # The font we have to get from umdimage2.
        font_dir = os.path.join(gfx_dir, "font")
        if not os.path.isdir(font_dir):
            os.makedirs(font_dir)

        progress.setValue(1)
        # And convert to PNG with an alpha channel so our editor can use it.
        font1 = font_bmp_to_alpha(os.path.join(self.data01_dir, "jp", "font", "font.pak", "0000.bmp"))
        progress.setValue(2)
        font2 = font_bmp_to_alpha(os.path.join(self.data01_dir, "jp", "font", "font.pak", "0002.bmp"))
        progress.setValue(3)

        font1.save(os.path.join(font_dir, "Font01.png"))
        font2.save(os.path.join(font_dir, "Font02.png"))
        shutil.copy(
            os.path.join(self.data01_dir, "jp", "font", "font.pak", "0001.font"), os.path.join(font_dir, "Font01.font")
        )
        shutil.copy(
            os.path.join(self.data01_dir, "jp", "font", "font.pak", "0003.font"), os.path.join(font_dir, "Font02.font")
        )

        progress.setValue(4)

        # And then the flash files. This'll be fun.
        flash_dir = os.path.join(gfx_dir, "flash")
        if not os.path.isdir(flash_dir):
            os.makedirs(flash_dir)

        # Because there's so many in so many different places, I just stored a list
        # of the flash files we need in the gfx_base archive. So let's load that.
        with open(os.path.join(gfx_dir, "fla.txt"), "rb") as fla:
            fla_list = fla.readlines()

            progress.setLabelText("Copying flash.")
            progress.setMaximum(len(fla_list))
            progress.setValue(0)

            for i, flash in enumerate(fla_list):
                if i % 10 == 0:
                    progress.setValue(i)

                if progress.wasCanceled():
                    return

                flash = flash.strip()
                fla_name = flash[:7]  # fla_###

                src = os.path.join(self.data01_dir, "all", "flash", flash)
                dest = os.path.join(flash_dir, "%s.gim" % fla_name)

                shutil.copy(src, dest)

            progress.setValue(len(fla_list))

        # We have a couple sets of files that aren't named the way we want them to
        # be, just because of how they're stored in umdimage.
        progress.setLabelText("Renaming files.")
        to_rename = [("movies", "movie_%03d.gim", range(32)), ("nametags", "%02d.gim", range(23) + [24, 25, 30, 31])]

        for (folder, pattern, nums) in to_rename:
            folder = os.path.join(gfx_dir, folder)
            files = glob.glob(os.path.join(folder, "*.gim"))

            progress.setMaximum(len(files))
            progress.setValue(0)

            for i, image in enumerate(files):
                if i % 10 == 0:
                    progress.setValue(i)

                if progress.wasCanceled():
                    return

                src = image
                dest = os.path.join(folder, pattern % nums[i])

                if os.path.isfile(dest):
                    os.remove(dest)

                shutil.move(src, dest)

        sprite_dir = os.path.join(gfx_dir, "sprites")
        gmo_files = glob.glob(os.path.join(sprite_dir, "*.gmo"))

        progress.setLabelText("Extracting GMO files.")
        progress.setValue(0)
        progress.setMaximum(len(gmo_files))

        for i, gmo_file in enumerate(gmo_files):
            if i % 10 == 0:
                progress.setValue(i)

            if progress.wasCanceled():
                return

            name, ext = os.path.splitext(os.path.basename(gmo_file))
            gim_file = os.path.join(sprite_dir, name + ".gim")

            gmo = GmoFile(filename=gmo_file)

            # Once we've loaded it, we're all done with it, so make it go away.
            os.remove(gmo_file)

            if gmo.gim_count() == 0:
                continue

            gim = gmo.get_gim(0)

            with open(gim_file, "wb") as f:
                gim.tofile(f)

        if self.ui.chkGimToPng.isChecked():
            gim_files = glob.glob(os.path.join(gfx_dir, "*", "*.gim"))

            progress.setLabelText("Converting GIM to PNG.")
            progress.setValue(0)
            progress.setMaximum(len(gim_files))

            converter = GimConverter()

            for i, gim_file in enumerate(gim_files):
                progress.setValue(i)
                if progress.wasCanceled():
                    return

                converter.gim_to_png(gim_file)
                os.remove(gim_file)

        progress.close()

        self.gfx_dir = gfx_dir

        self.ui.grpStep5.setEnabled(False)
        self.ui.grpStep6.setEnabled(True)
Beispiel #7
0
class GmoFile():
    def __init__(self, data=None, offset=0, filename=None):
        self.data = None
        self.__gim_files = []

        self.gimconv = GimConverter()

        if not data == None:
            self.load_data(data, offset)
        elif not filename == None:
            self.load_file(filename)

    def load_file(self, filename):
        data = BitStream(filename=filename)
        self.load_data(data)

    def load_data(self, data, offset=0):
        if not data[offset * 8:offset * 8 + GMO_MAGIC.len] == GMO_MAGIC:
            _LOGGER.error("GMO header not found at 0x%04X." % offset)
            return

        data.bytepos = offset + GMO_SIZE_OFFSET
        gmo_size = data.read("uintle:32") + GMO_SIZE_DIFF

        self.data = BitStream(data[offset * 8:(offset + gmo_size) * 8])

        self.__find_gims()

    def save(self, filename):
        with open(filename, "wb") as f:
            self.data.tofile(f)

    def __find_gims(self):
        if self.data == None:
            return

        self.__gim_files = []

        for gim_start in self.data.findall(GIM_MAGIC, bytealigned=True):
            gim_size_pos = gim_start + (GIM_SIZE_OFFSET * 8)  # Bit pos.
            gim_size = self.data[gim_size_pos:gim_size_pos +
                                 32].uintle + GIM_SIZE_DIFF

            # And turn it into a byte position.
            gim_start /= 8
            self.__gim_files.append((gim_start, gim_size))

    def gim_count(self):
        return len(self.__gim_files)

    def get_gim(self, gim_id):
        if gim_id >= self.gim_count():
            raise GimIndexError("Invalid GIM ID.")

        gim_start, gim_size = self.__gim_files[gim_id]
        gim_data = self.data[gim_start * 8:(gim_start + gim_size) * 8]

        return gim_data

    def replace_png_file(self, gim_id, filename, quantize_to_fit=True):

        if quantize_to_fit:
            quantize_order = [
                QuantizeType.auto, QuantizeType.index8, QuantizeType.index4
            ]
        else:
            quantize_order = [QuantizeType.auto]
        quantize_id = 0

        (fd, temp_gim) = tempfile.mkstemp(suffix=".gim", prefix="sdse-")
        os.close(fd)  # Don't need the open file handle.

        while True:
            self.gimconv.png_to_gim(filename, temp_gim,
                                    quantize_order[quantize_id])

            try:
                self.replace_gim_file(gim_id, temp_gim)
            except GimSizeError:
                quantize_id += 1
            except GimIndexError:
                os.remove(temp_gim)
                raise
            else:
                # If we didn't except, that means we succeeded, so we can leave.
                _LOGGER.debug("Quantized PNG to %s" %
                              quantize_order[quantize_id])
                break

            if quantize_id > len(quantize_order):
                _LOGGER.error(
                    "Unable to convert %s into a GIM small enough to insert." %
                    filename)
                break

        os.remove(temp_gim)

    def replace_gim_file(self, gim_id, filename):
        gim_data = BitStream(filename=filename)
        self.replace_gim(gim_id, gim_data)

    def replace_gim(self, gim_id, gim_data):
        if gim_id >= self.gim_count():
            raise GimIndexError("Invalid GIM ID.")

        gim_start, gim_size = self.__gim_files[gim_id]

        if gim_data.len / 8 > gim_size:
            raise GimSizeError("GIM too large. %d bytes > %d bytes" %
                               (gim_data.len / 8, gim_size))
            # return

        self.data.overwrite(gim_data, gim_start * 8)

        # Leave the length alone, though, because we know we have that much space
        # to work with from the original GIM file that was there, and there's no
        # point in shrinking that down if someone happens to want to re-replace
        # this GIM file without reloading the whole thing.

    def extract(self, directory, to_png=False):
        if not os.path.isdir(directory):
            os.makedirs(directory)

        for id in range(self.gim_count()):
            gim = self.get_gim(id)

            out_gim = os.path.join(directory, "%04d.gim" % id)
            out_png = os.path.join(directory, "%04d.png" % id)

            with open(out_gim, "wb") as f:
                gim.tofile(f)

            if to_png:
                self.gimconv.gim_to_png(out_gim, out_png)
                os.remove(out_gim)