def walker_hash(self, ):
start_dir = QtWidgets.QFileDialog.getExistingDirectory(
self,
'Game Root folder',
self.cfg.get("dir_ovls_in", "C://"),
)
hash_dict = {}
if start_dir:
# don't use internal data
ovl_data = OvlFile()
error_files = []
ovl_files = walker.walk_type(start_dir, extension="ovl")
of_max = len(ovl_files)
for of_index, ovl_path in enumerate(ovl_files):
self.update_progress("Hashing names: " +
os.path.basename(ovl_path),
value=of_index,
vmax=of_max)
try:
# read ovl file
new_hashes = ovl_data.load(
ovl_path, commands=("generate_hash_table", ))
hash_dict.update(new_hashes)
except:
error_files.append(ovl_path)
if error_files:
print(f"{error_files} caused errors!")
# write the hash text file to the hashes folder
export_dir = os.path.join(os.getcwd(), "hashes")
out_path = os.path.join(export_dir,
f"{os.path.basename(start_dir)}.txt")
with open(out_path, "w") as f:
for k, v in hash_dict.items():
f.write(f"{k} = {v}\n")
print(f"Wrote {len(hash_dict)} items to {out_path}")
def bulk_extract_ovls(errors, export_dir, gui, start_dir, only_types):
# don't use internal data
ovl_data = OvlFile()
error_files = []
ovl_files = walk_type(start_dir, extension=".ovl")
of_max = len(ovl_files)
for of_index, ovl_path in enumerate(ovl_files):
gui.update_progress("Walking OVL files: " + os.path.basename(ovl_path),
value=of_index,
vmax=of_max)
try:
# read ovl file
ovl_data.load(ovl_path, commands=gui.commands)
# create an output folder for it
rel_p = os.path.relpath(ovl_path, start=start_dir)
rel_d = os.path.splitext(rel_p)[0]
outdir = os.path.join(export_dir, rel_d)
out_paths, error_files_new = ovl_data.extract(
outdir, only_types=only_types)
error_files += error_files_new
except Exception as ex:
traceback.print_exc()
errors.append((ovl_path, ex))
interaction.extract_error_warning(error_files)
class TestDirEntries(unittest.TestCase):
# load an empty ovl file for each test case
def setUp(self):
self.ovlfile = OvlFile()
self.ovlfile.load('tests/Data/empty.ovl')
def test_ovl_no_included_ovls(self):
self.assertEqual(len(self.ovlfile.included_ovls), 0,
"Should have no included_ovls")
def test_inject_dir(self):
self.assertEqual(len(self.ovlfile.included_ovls), 0,
"Should have no included_ovls")
self.ovlfile.add_included_ovl('test1')
self.assertEqual(len(self.ovlfile.included_ovls), 1,
"Should have one included_ovl")
self.assertEqual(self.ovlfile.included_ovls[0].name, "test1",
"should have included_ovl 1 name 'test1'")
self.ovlfile.add_included_ovl('test2')
self.assertEqual(len(self.ovlfile.included_ovls), 2,
"Should have two included_ovl")
self.assertEqual(self.ovlfile.included_ovls[1].name, "test2",
"should have included_ovl 2 as 'test2'")
# try adding a existing included_ovl
self.ovlfile.add_included_ovl('test1')
self.assertEqual(len(self.ovlfile.included_ovls), 2,
"Should have two included_ovl")
def test_remove_dir(self):
self.assertEqual(len(self.ovlfile.included_ovls), 0,
"Should have no included_ovls")
self.ovlfile.add_included_ovl('test1')
self.assertEqual(len(self.ovlfile.included_ovls), 1,
"Should have one included_ovl")
self.assertEqual(self.ovlfile.included_ovls[0].name, "test1",
"should have included_ovl 1 as 'test1'")
self.ovlfile.add_included_ovl('test2')
self.assertEqual(len(self.ovlfile.included_ovls), 2,
"Should have two included_ovl")
self.assertEqual(self.ovlfile.included_ovls[1].name, "test2",
"should have included_ovl 2 as 'test2'")
# remove existing included_ovl
self.ovlfile.remove_included_ovl('test1')
self.assertEqual(len(self.ovlfile.included_ovls), 1,
"Should have one included_ovl")
self.assertEqual(self.ovlfile.included_ovls[0].name, "test2",
"should have included_ovl 1 as 'test2'")
# remove non-existing dir
self.ovlfile.remove_included_ovl('test3')
self.assertEqual(len(self.ovlfile.included_ovls), 1,
"Should have one included_ovl")
self.assertEqual(self.ovlfile.included_ovls[0].name, "test2",
"should have included_ovl 1 as 'test2'")
def test_rename_dir(self):
self.assertEqual(len(self.ovlfile.included_ovls), 0,
"Should have no included_ovls")
self.ovlfile.add_included_ovl('test1')
self.assertEqual(len(self.ovlfile.included_ovls), 1,
"Should have one included_ovl")
self.assertEqual(self.ovlfile.included_ovls[0].name, "test1",
"should have included_ovl 1 as 'test1'")
self.ovlfile.add_included_ovl('test2')
self.assertEqual(len(self.ovlfile.included_ovls), 2,
"Should have two included_ovl")
self.assertEqual(self.ovlfile.included_ovls[1].name, "test2",
"should have included_ovl 2 as 'test2'")
# try renaming an existing included_ovl
self.ovlfile.rename_included_ovl('test1', 'test3')
self.assertEqual(len(self.ovlfile.included_ovls), 2,
"Should have two included_ovl")
self.assertEqual(self.ovlfile.included_ovls[0].name, "test3",
"should have included_ovl 1 as 'test3'")
# try renaming a missing included_ovl
self.ovlfile.rename_included_ovl('test1', 'test5')
self.assertEqual(len(self.ovlfile.included_ovls), 2,
"Should have two included_ovl")
self.assertEqual(self.ovlfile.included_ovls[0].name, "test3",
"should have included_ovl 1 as 'test3'")
"""
class MainWindow(widgets.MainWindow):
def __init__(self):
widgets.MainWindow.__init__(
self,
"OVL Archive Editor",
)
self.resize(800, 600)
self.ovl_data = OvlFile(progress_callback=self.update_progress)
self.filter = "Supported files ({})".format(" ".join(
"*" + t for t in extract.SUPPORTED_TYPES))
self.file_widget = widgets.FileWidget(self, self.cfg)
self.file_widget.setToolTip(
"The name of the OVL file that is currently open.")
self.p_action = QtWidgets.QProgressBar(self)
self.p_action.setGeometry(0, 0, 200, 15)
self.p_action.setTextVisible(True)
self.p_action.setMaximum(1)
self.p_action.setValue(0)
self.t_action_current_message = "No operation in progress"
self.t_action = QtWidgets.QLabel(self,
text=self.t_action_current_message)
self.game_container = widgets.LabelCombo("Game:", games)
self.game_container.entry.currentIndexChanged.connect(
self.game_changed)
self.game_container.entry.setEditable(False)
header_names = ["Name", "File Type", "DJB", "Unk0", "Unk1"]
self.files_container = widgets.SortableTable(header_names, self)
self.dir_container = widgets.EditCombo(self)
# toggles
self.t_show_temp_files = QtWidgets.QCheckBox("Save Temp Files")
self.t_show_temp_files.setToolTip(
"By default, temporary files are converted to usable ones and back on the fly."
)
self.t_show_temp_files.setChecked(False)
self.t_2K = QtWidgets.QCheckBox("Inject 2K")
self.t_2K.setToolTip(
"Experimental: Increase a JWE Diffuse or Normal map to 2048x2048 resolution."
)
self.t_2K.setChecked(False)
self.sp_hash = QtWidgets.QCheckBox("New Species Hash")
self.sp_hash.setToolTip("Experimental")
self.sp_hash.setChecked(False)
self.ext_dat = QtWidgets.QCheckBox("Use External DAT")
self.ext_dat.setToolTip(
"Experimental: Save the ovl with an external STATIC DAT instead of one in memory"
)
self.ext_dat.setChecked(False)
self.ext_dat.stateChanged.connect(self.dat_show)
self.dat_widget = widgets.FileWidget(self,
self.cfg,
ask_user=False,
dtype="DAT",
poll=False)
self.dat_widget.setToolTip("External .dat file path")
self.dat_widget.hide()
self.e_name_pairs = [(QtWidgets.QLineEdit("old"),
QtWidgets.QLineEdit("new")) for i in range(1)]
self.t_write_dat = QtWidgets.QCheckBox("Save DAT")
self.t_write_dat.setToolTip(
"Writes decompressed archive streams to DAT files for debugging.")
self.t_write_dat.setChecked(False)
self.t_write_dat.stateChanged.connect(self.load)
self.t_write_frag_log = QtWidgets.QCheckBox("Save Frag Log")
self.t_write_frag_log.setToolTip("For devs.")
self.t_write_frag_log.setChecked(False)
self.t_write_frag_log.stateChanged.connect(self.load)
self.qgrid = QtWidgets.QGridLayout()
self.qgrid.addWidget(self.file_widget, 0, 0, 1, 5)
self.qgrid.addWidget(self.t_show_temp_files, 1, 0)
self.qgrid.addWidget(self.t_write_dat, 1, 1)
self.qgrid.addWidget(self.t_write_frag_log, 1, 2)
self.qgrid.addWidget(self.ext_dat, 1, 3)
self.qgrid.addWidget(self.sp_hash, 1, 4)
for (old, new) in self.e_name_pairs:
self.qgrid.addWidget(old, 2, 0, 1, 2)
self.qgrid.addWidget(new, 2, 2, 1, 2)
self.qgrid.addWidget(
self.game_container,
2,
4,
)
self.qgrid.addWidget(self.files_container, 3, 0, 1, 5)
self.qgrid.addWidget(self.dir_container, 4, 0, 1, 5)
self.qgrid.addWidget(self.p_action, 5, 0, 1, 5)
self.qgrid.addWidget(self.t_action, 6, 0, 1, 5)
self.qgrid.addWidget(self.dat_widget, 7, 0, 1, 5)
self.central_widget.setLayout(self.qgrid)
mainMenu = self.menuBar()
fileMenu = mainMenu.addMenu('File')
editMenu = mainMenu.addMenu('Edit')
helpMenu = mainMenu.addMenu('Help')
button_data = (
(fileMenu, "New", self.file_widget.ask_open_dir, "CTRL+N", "new"),
(fileMenu, "Open", self.file_widget.ask_open, "CTRL+O", "dir"),
(fileMenu, "Save", self.save_ovl, "CTRL+S", "save"),
(fileMenu, "Exit", self.close, "", "exit"),
(editMenu, "Unpack", self.extract_all, "CTRL+U", "extract"),
(editMenu, "Inject", self.inject, "CTRL+I", "inject"),
(editMenu, "Rename", self.hasher, "CTRL+R", ""),
(editMenu, "Dat Edit", self.dat_replacement, "CTRL+J", ""),
(editMenu, "Remove Selected", self.remover, "DEL", ""),
(editMenu, "Walk", self.walker, "", ""),
# (editMenu, "Reload", self.reload, "", ""),
(editMenu, "Generate Hash Table", self.walker_hash, "", ""),
(helpMenu, "Report Bug", self.report_bug, "", "report"),
(helpMenu, "Documentation", self.online_support, "", "manual"))
self.add_to_menu(button_data)
self.check_version()
self.load_hash_table()
def game_changed(self, ):
game = self.game_container.entry.currentText()
set_game(self.ovl_data, game)
@property
def commands(self, ):
# get those commands that are set to True
return [x for x in ("write_dat", "write_frag_log") if getattr(self, x)]
@property
def show_temp_files(self, ):
return self.t_show_temp_files.isChecked()
@property
def write_2K(self, ):
return self.t_2K.isChecked()
@property
def species_hash(self, ):
return self.sp_hash.isChecked()
@property
def use_ext_dat(self, ):
return self.ext_dat.isChecked()
@property
def write_dat(self, ):
return self.t_write_dat.isChecked()
@property
def write_frag_log(self, ):
return self.t_write_frag_log.isChecked()
def dat_show(self, ):
if self.use_ext_dat:
self.dat_widget.show()
else:
self.dat_widget.hide()
def update_commands(self):
# at some point, just set commands to archive and trigger changes there
if self.file_widget.filename:
self.ovl_data.commands = self.commands
def update_progress(self, message, value=None, vmax=None):
# avoid gui updates if the value won't actually change the percentage.
# this saves us from making lots of GUI update calls that don't really
# matter.
try:
if vmax > 100 and (value % (vmax // 100)) and value != 0:
value = None
except ZeroDivisionError:
value = 0
except TypeError:
value = None
# update progress bar values if specified
if value is not None:
self.p_action.setValue(value)
if vmax is not None:
self.p_action.setMaximum(vmax)
# don't update the GUI unless the message has changed. label updates
# are expensive
if self.t_action_current_message != message:
self.t_action.setText(message)
self.t_action_current_message = message
def load_hash_table(self):
print("Loading hash table...")
start_time = time.time()
self.hash_table = {}
hashes_dir = os.path.join(os.getcwd(), "hashes")
try:
for file in os.listdir(hashes_dir):
self.read_table(os.path.join(hashes_dir, file),
self.hash_table,
int_key=True)
except:
pass
# print(self.hash_table)
print(
f"Loaded {len(self.hash_table)} hash - name pairs in {time.time() - start_time:.2f} seconds."
)
def show_dependencies(self, file_index):
file_entry = self.ovl_data.files[file_index]
# print(file_entry)
ss_entry = self.ovl_data.ss_dict[file_entry.name]
# print(ss_entry)
ss_p = ss_entry.pointers[0]
# print(file_entry.dependencies)
logging.debug(
f"File: {ss_p.pool_index} {ss_p.data_offset} {ss_entry.name}")
try:
for dep in file_entry.dependencies:
p = dep.pointers[0]
p.data_size = 8
# the index goes into the flattened list of pools
p.read_data(self.ovl_data.pools)
assert p.data == b'\x00\x00\x00\x00\x00\x00\x00\x00'
logging.debug(
f"Dependency: {p.pool_index} {p.data_offset} {dep.name}")
for f in ss_entry.fragments:
p0 = f.pointers[0]
p1 = f.pointers[1]
logging.debug(
f"Fragment: {p0.pool_index} {p0.data_offset} {p1.pool_index} {p1.data_offset}"
)
except:
logging.error(f"Dependency failed {file_entry.dependencies}")
@staticmethod
def read_table(fp, dic, int_key=False):
if fp.endswith(".txt"):
with open(fp, "r") as f:
for line in f:
line = line.strip()
if line:
k, v = line.split(" = ")
if int_key:
dic[int(k)] = v
else:
dic[k] = v
def load(self):
if self.file_widget.filepath:
self.file_widget.dirty = False
self.update_progress("Reading OVL " + self.file_widget.filepath,
value=0,
vmax=0)
try:
self.ovl_data.load(self.file_widget.filepath,
commands=self.commands,
hash_table=self.hash_table)
self.ovl_data.load_archives()
# print(self.ovl_data)
except Exception as ex:
traceback.print_exc()
interaction.showdialog(str(ex))
self.update_gui_table()
game = get_game(self.ovl_data)
self.game_container.entry.setText(game)
def create_ovl(self, ovl_dir):
# clear the ovl
self.ovl_data = OvlFile(progress_callback=self.update_progress)
self.game_changed()
# read tables for constants
mimes_table = {}
tables_dir = os.path.join(os.getcwd(), "dicts")
self.read_table(os.path.join(tables_dir, "mimes.txt"), mimes_table)
try:
self.ovl_data.create(ovl_dir, mime_names_dict=mimes_table)
except Exception as ex:
traceback.print_exc()
interaction.showdialog(str(ex))
self.update_gui_table()
def is_open_ovl(self):
if not self.file_widget.filename:
interaction.showdialog("You must open an OVL file first!")
else:
return True
def update_gui_table(self, ):
start_time = time.time()
print(f"Loading {len(self.ovl_data.files)} files into gui...")
self.files_container.set_data([(f.name, f.ext, f.file_hash, f.unkn_0,
f.unkn_1)
for f in self.ovl_data.files])
self.dir_container.set_data(self.ovl_data.dir_names)
print(f"Loaded GUI in {time.time() - start_time:.2f} seconds!")
self.update_progress("Operation completed!", value=1, vmax=1)
def save_ovl(self):
if self.is_open_ovl():
file_src = QtWidgets.QFileDialog.getSaveFileName(
self,
'Save OVL',
os.path.join(self.cfg.get("dir_ovls_out", "C://"),
self.file_widget.filename),
"OVL files (*.ovl)",
)[0]
if file_src:
self.cfg["dir_ovls_out"], ovl_name = os.path.split(file_src)
try:
self.ovl_data.save(file_src, self.use_ext_dat,
self.dat_widget.filepath)
self.file_widget.dirty = False
self.update_progress("Operation completed!",
value=1,
vmax=1)
except BaseException as ex:
traceback.print_exc()
interaction.showdialog(str(ex))
def extract_all(self):
if self.is_open_ovl():
out_dir = QtWidgets.QFileDialog.getExistingDirectory(
self,
'Output folder',
self.cfg.get("dir_extract", "C://"),
)
if out_dir:
self.cfg["dir_extract"] = out_dir
try:
out_paths, error_files, skip_files = self.ovl_data.extract(
out_dir, self.show_temp_files)
interaction.skip_messages(error_files, skip_files)
except Exception as ex:
traceback.print_exc()
interaction.showdialog(str(ex))
def inject(self):
if self.is_open_ovl():
files = QtWidgets.QFileDialog.getOpenFileNames(
self, 'Inject files', self.cfg.get("dir_inject", "C://"),
self.filter)[0]
if files:
self.cfg["dir_inject"] = os.path.dirname(files[0])
try:
inject.inject(self.ovl_data, files, self.show_temp_files,
self.write_2K, self.update_progress)
self.file_widget.dirty = True
except Exception as ex:
traceback.print_exc()
interaction.showdialog(str(ex))
def hasher(self):
if self.is_open_ovl():
names = [(tup[0].text(), tup[1].text())
for tup in self.e_name_pairs]
hasher.rename(self.ovl_data, names, species_mode=self.species_hash)
self.update_gui_table()
def dat_replacement(self):
if self.is_open_ovl():
names = [(tup[0].text(), tup[1].text())
for tup in self.e_name_pairs]
if self.species_hash:
hasher.species_dat_replacer(self.ovl_data, names)
else:
hasher.dat_replacer(self.ovl_data, names)
self.update_gui_table()
# reload modules, debug feature, allows reloading extraction modules without restarting the gui
# modules need to be imported completely, import xxxx, from xxx import yyy will not work.
# def reload(self):
# reload(modules.formats.SPECDEF)
# reload(modules.extract)
def remover(self):
if self.is_open_ovl():
selected_file_names = self.files_container.table.get_selected_files(
)
if selected_file_names:
try:
remover.file_remover(self.ovl_data, selected_file_names)
except:
traceback.print_exc()
self.update_gui_table()
def walker_hash(self, ):
start_dir = QtWidgets.QFileDialog.getExistingDirectory(
self,
'Game Root folder',
self.cfg.get("dir_ovls_in", "C://"),
)
hash_dict = {}
if start_dir:
# don't use internal data
ovl_data = OvlFile()
error_files = []
ovl_files = walker.walk_type(start_dir, extension="ovl")
of_max = len(ovl_files)
for of_index, ovl_path in enumerate(ovl_files):
self.update_progress("Hashing names: " +
os.path.basename(ovl_path),
value=of_index,
vmax=of_max)
try:
# read ovl file
new_hashes = ovl_data.load(
ovl_path, commands=("generate_hash_table", ))
hash_dict.update(new_hashes)
except:
error_files.append(ovl_path)
if error_files:
print(f"{error_files} caused errors!")
# write the hash text file to the hashes folder
export_dir = os.path.join(os.getcwd(), "hashes")
out_path = os.path.join(export_dir,
f"{os.path.basename(start_dir)}.txt")
with open(out_path, "w") as f:
for k, v in hash_dict.items():
f.write(f"{k} = {v}\n")
print(f"Wrote {len(hash_dict)} items to {out_path}")
def walker(self):
start_dir = QtWidgets.QFileDialog.getExistingDirectory(
self,
'Game Root folder',
self.cfg.get("dir_ovls_in", "C://"),
)
walker.bulk_test_models(self, start_dir)
def closeEvent(self, event):
if self.file_widget.dirty:
quit_msg = f"Quit? You will lose unsaved work on {os.path.basename(self.file_widget.filepath)}!"
if not interaction.showdialog(quit_msg, ask=True):
event.ignore()
return
event.accept()
@staticmethod
def check_version():
is_64bits = sys.maxsize > 2**32
if not is_64bits:
interaction.showdialog(
"Either your operating system or your python installation is not 64 bits.\n"
"Large OVLs will crash unexpectedly!")
if sys.version_info[0] != 3 or sys.version_info[1] < 7 or (
sys.version_info[1] == 7 and sys.version_info[2] < 6):
interaction.showdialog("Python 3.7.6+ x64 bit is expected!")
def bulk_test_models(gui, start_dir, walk_ovls=True, walk_models=True):
errors = []
if start_dir:
export_dir = os.path.join(start_dir, "walker_export")
# don't use internal data
ovl_data = OvlFile()
mdl2_data = Mdl2File()
if walk_ovls:
error_files = []
skip_files = []
ovl_files = walk_type(start_dir, extension="ovl")
of_max = len(ovl_files)
for of_index, ovl_path in enumerate(ovl_files):
gui.update_progress("Walking OVL files: " +
os.path.basename(ovl_path),
value=of_index,
vmax=of_max)
try:
# read ovl file
ovl_data.load(ovl_path, commands=gui.commands)
ovl_data.load_archives()
# create an output folder for it
outdir = os.path.join(export_dir,
os.path.basename(ovl_path[:-4]))
out_paths, error_files_new, skip_files_new = ovl_data.extract(
outdir, only_types=(".ms2", ))
error_files += error_files_new
skip_files += skip_files_new
except Exception as ex:
traceback.print_exc()
errors.append((ovl_path, ex))
interaction.skip_messages(error_files, skip_files)
# holds different types of flag - list of byte maps pairs
type_dic = {}
if walk_models:
mdl2_files = walk_type(export_dir, extension="mdl2")
mf_max = len(mdl2_files)
for mf_index, mdl2_path in enumerate(mdl2_files):
mdl2_name = os.path.basename(mdl2_path)
gui.update_progress("Walking MDL2 files: " + mdl2_name,
value=mf_index,
vmax=mf_max)
try:
mdl2_data.load(mdl2_path, quick=True, map_bytes=True)
for model in mdl2_data.models:
if model.flag not in type_dic:
type_dic[model.flag] = ([], [])
type_dic[model.flag][0].append(mdl2_name)
type_dic[model.flag][1].append(
(model.bytes_mean, model.bytes_max,
model.bytes_min))
except Exception as ex:
traceback.print_exc()
errors.append((mdl2_path, ex))
# report
print("\nThe following errors occured:")
for file_path, ex in errors:
print(file_path, str(ex))
print("\nThe following type - map pairs were found:")
for flag, tup in sorted(type_dic.items()):
print(flag, bin(flag))
names, maps_list = tup
print("Some files:", list(set(names))[:25])
print("num models", len(maps_list))
means, maxs, mins = zip(*maps_list)
print(len(means))
print("mean", np.mean(means, axis=0).astype(dtype=np.ubyte))
print("max", np.max(maxs, axis=0))
print("min", np.min(mins, axis=0))
print()
gui.update_progress("Operation completed!", value=1, vmax=1)
def generate_hash_table(gui, start_dir):
hash_dict = {}
if start_dir:
# don't use internal data
ovl_data = OvlFile()
dic = {}
lists = {
"mimes":
("name", "mime_hash", "mime_version", "triplet_count", "triplets"),
"files": ("unkn_0", "unkn_1")
}
for list_name, attr_names in lists.items():
dic[list_name] = {}
for attr_name in attr_names:
dic[list_name][attr_name] = {}
error_files = []
ovl_files = walk_type(start_dir, extension=".ovl")
of_max = len(ovl_files)
for of_index, ovl_path in enumerate(ovl_files):
gui.update_progress("Hashing names: " + os.path.basename(ovl_path),
value=of_index,
vmax=of_max)
try:
# read ovl file
new_hashes = ovl_data.load(ovl_path,
commands=("generate_hash_table", ))
for list_name, attr_names in lists.items():
for entry in getattr(ovl_data, list_name):
for attr_name in attr_names:
v = getattr(entry, attr_name)
if attr_name == "triplets":
v = [(t.a, t.b, t.c) for t in v]
# if the value already exists, make sure it is indeed constant (for this version)
if entry.ext in dic[list_name][attr_name]:
if v != dic[list_name][attr_name][entry.ext]:
logging.error(
f"{list_name}.{attr_name} is not constant for {entry.ext}! ({v} vs. {dic[list_name][attr_name][entry.ext]})"
)
dic[list_name][attr_name][entry.ext] = v
hash_dict.update(new_hashes)
except:
error_files.append(ovl_path)
# print(dic)
if error_files:
logging.error(f"{error_files} caused errors!")
try:
# write the hash text file to the hashes folder
export_dir = os.path.join(os.getcwd(), "hashes")
out_path = os.path.join(export_dir,
f"{os.path.basename(start_dir)}.txt")
with open(out_path, "w") as f:
for k, v in hash_dict.items():
f.write(f"{k} = {v}\n")
out_path = os.path.join(
export_dir, f"constants_{os.path.basename(start_dir)}.py")
with open(out_path, "w") as f:
for list_name, attr_names in lists.items():
for attr_name in attr_names:
f.write(
f"{list_name}_{attr_name} = {dic[list_name][attr_name]}\n\n"
)
except BaseException as err:
print(err)
logging.info(f"Wrote {len(hash_dict)} items to {out_path}")
class MainWindow(widgets.MainWindow):
def __init__(self):
widgets.MainWindow.__init__(
self,
"OVL Archive Editor",
)
self.resize(800, 600)
self.ovl_data = OvlFile(progress_callback=self.update_progress)
self.ovl_data.load_hash_table()
self.filter = "Supported files ({})".format(" ".join(
"*" + t for t in SUPPORTED_TYPES))
self.file_widget = widgets.FileWidget(self, self.cfg)
self.file_widget.setToolTip(
"The name of the OVL file that is currently open")
self.p_action = QtWidgets.QProgressBar(self)
self.p_action.setGeometry(0, 0, 200, 15)
self.p_action.setTextVisible(True)
self.p_action.setMaximum(1)
self.p_action.setValue(0)
self.t_action_current_message = "No operation in progress"
self.t_action = QtWidgets.QLabel(self,
text=self.t_action_current_message)
self.game_choice = widgets.LabelCombo("Game:",
[g.value for g in games])
# only listen to user changes
self.game_choice.entry.textActivated.connect(self.game_changed)
self.game_choice.entry.setEditable(False)
self.compression_choice = widgets.LabelCombo(
"Compression:", [c.name for c in Compression])
# only listen to user changes
self.compression_choice.entry.textActivated.connect(
self.compression_changed)
self.compression_choice.entry.setEditable(False)
header_names = ["Name", "File Type", "DJB"]
self.model = QtWidgets.QFileSystemModel()
self.dirs_container = QtWidgets.QTreeView()
self.dirs_container.setModel(self.model)
self.dirs_container.setColumnHidden(1, True)
self.dirs_container.setColumnHidden(2, True)
self.dirs_container.setColumnHidden(3, True)
self.dirs_container.doubleClicked.connect(self.dirs_clicked)
self.set_game_dir()
self.dirs_container.header().setSortIndicator(0,
QtCore.Qt.AscendingOrder)
self.dirs_container.model().sort(
self.dirs_container.header().sortIndicatorSection(),
self.dirs_container.header().sortIndicatorOrder())
self.dirs_container.setAnimated(False)
self.dirs_container.setIndentation(20)
self.dirs_container.setSortingEnabled(True)
self.dirs_container.setWindowTitle("Dir View")
self.dirs_container.resize(640, 480)
# create the table
self.files_container = widgets.SortableTable(header_names,
IGNORE_TYPES)
# connect the interaction functions
self.files_container.table.model.member_renamed.connect(
self.rename_handle)
self.files_container.table.files_dragged.connect(self.drag_files)
self.files_container.table.files_dropped.connect(self.inject_files)
# self.files_container.table.file_selected.connect(self.show_dependencies)
self.included_ovls_view = widgets.EditCombo(self)
self.included_ovls_view.setToolTip(
"These OVL files are loaded by the current OVL file, so their files are included"
)
self.included_ovls_view.entries_changed.connect(
self.ovl_data.set_included_ovl_names)
self.dat_widget = widgets.FileWidget(self,
self.cfg,
ask_user=False,
dtype="DAT",
poll=False)
self.dat_widget.setToolTip(
"External .dat file path to overwrite internal OVS data")
self.dat_widget.hide()
right_frame = QtWidgets.QWidget()
hbox = QtWidgets.QVBoxLayout()
hbox.addWidget(self.file_widget)
hbox.addWidget(self.files_container)
hbox.addWidget(self.included_ovls_view)
hbox.addWidget(self.dat_widget)
right_frame.setLayout(hbox)
# toggles
self.t_show_temp_files = QtWidgets.QCheckBox("Save Temp Files")
self.t_show_temp_files.setToolTip(
"By default, temporary files are converted to usable ones and back on the fly"
)
self.t_show_temp_files.setChecked(False)
self.in_folder = QtWidgets.QCheckBox("Process Folder")
self.in_folder.setToolTip(
"Runs commands on all OVLs of current folder")
self.in_folder.setChecked(False)
self.ext_dat = QtWidgets.QCheckBox("Use External DAT")
self.ext_dat.setToolTip(
"Experimental: Save the ovl with an external STATIC DAT instead of one in memory"
)
self.ext_dat.setChecked(False)
self.ext_dat.stateChanged.connect(self.dat_show)
self.t_animal_ovl = QtWidgets.QCheckBox("Animal OVL Mode")
self.t_animal_ovl.setToolTip(
"Renames only MS2, MDL2 and MOTIONGRAPH files.")
self.t_animal_ovl.setChecked(False)
self.t_unsafe = QtWidgets.QCheckBox("Unsafe Mode")
self.t_unsafe.setToolTip(
"Forces unsafe (brute force) replacement. May break your files.")
self.t_unsafe.setChecked(False)
self.e_name_old = QtWidgets.QTextEdit("old")
self.e_name_new = QtWidgets.QTextEdit("new")
self.e_name_old.setFixedHeight(100)
self.e_name_new.setFixedHeight(100)
self.e_name_old.setTabChangesFocus(True)
self.e_name_new.setTabChangesFocus(True)
self.t_write_dat = QtWidgets.QCheckBox("Save DAT")
self.t_write_dat.setToolTip(
"Writes decompressed archive streams to DAT files for debugging")
self.t_write_dat.setChecked(False)
self.t_write_dat.stateChanged.connect(self.load)
self.splitter = QtWidgets.QSplitter(QtCore.Qt.Horizontal)
self.splitter.addWidget(self.dirs_container)
self.splitter.addWidget(right_frame)
self.splitter.setSizes([200, 400])
self.splitter.setSizePolicy(QtWidgets.QSizePolicy.Expanding,
QtWidgets.QSizePolicy.Expanding)
self.qgrid = QtWidgets.QGridLayout()
self.qgrid.addWidget(self.e_name_old, 0, 0, 5, 1)
self.qgrid.addWidget(self.e_name_new, 0, 1, 5, 1)
self.qgrid.addWidget(self.t_show_temp_files, 0, 3)
self.qgrid.addWidget(self.t_write_dat, 1, 3)
self.qgrid.addWidget(self.ext_dat, 2, 3)
self.qgrid.addWidget(self.in_folder, 3, 3)
self.qgrid.addWidget(
self.game_choice,
0,
4,
)
self.qgrid.addWidget(
self.compression_choice,
1,
4,
)
self.qgrid.addWidget(self.t_animal_ovl, 2, 4)
self.qgrid.addWidget(self.t_unsafe, 3, 4)
self.qgrid.addWidget(self.splitter, 5, 0, 1, 5)
self.qgrid.addWidget(self.p_action, 6, 0, 1, 5)
self.qgrid.addWidget(self.t_action, 7, 0, 1, 5)
self.central_widget.setLayout(self.qgrid)
main_menu = self.menuBar()
file_menu = main_menu.addMenu('File')
edit_menu = main_menu.addMenu('Edit')
util_menu = main_menu.addMenu('Util')
help_menu = main_menu.addMenu('Help')
button_data = ((file_menu, "New", self.file_widget.ask_open_dir,
"CTRL+N", "new"),
(file_menu, "Open", self.file_widget.ask_open, "CTRL+O",
"dir"), (file_menu, "Save", self.save_ovl, "CTRL+S",
"save"),
(file_menu, "Save As", self.save_as_ovl, "CTRL+SHIFT+S",
"save"), (file_menu, "Exit", self.close, "", "exit"),
(edit_menu, "Unpack", self.extract_all, "CTRL+U",
"extract"), (edit_menu, "Inject", self.inject_ask,
"CTRL+I", "inject"),
(edit_menu, "Rename", self.rename, "CTRL+R",
""), (edit_menu, "Rename Contents",
self.rename_contents, "CTRL+SHIFT+R", ""),
(edit_menu, "Remove Selected", self.remover, "DEL",
""), (util_menu, "Inspect Models", self.inspect_models,
"", ""), (util_menu, "Inspect FGMs",
self.walker_fgm, "", ""),
(util_menu, "Generate Hash Table", self.walker_hash, "",
""), (util_menu, "Save Frag Log",
self.ovl_data.dump_frag_log, "",
""), (util_menu, "Open Tools Dir",
self.open_tools_dir, "", ""),
(util_menu, "Export File List", self.save_file_list, "",
""), (util_menu, "Set Game Dir", self.ask_game_dir, "",
""), (util_menu, "Export included ovl list",
self.save_included_ovls, "", ""),
(help_menu, "Report Bug", self.report_bug, "",
"report"), (help_menu, "Documentation",
self.online_support, "", "manual"))
self.add_to_menu(button_data)
self.check_version()
def ask_game_dir(self):
dir_game = QtWidgets.QFileDialog.getExistingDirectory(
self, "Open game folder")
self.cfg["dir_game"] = dir_game
return dir_game
def set_game_dir(self):
dir_game = self.cfg.get("dir_game", "")
if not dir_game:
dir_game = self.ask_game_dir()
if dir_game:
rt_index = self.model.setRootPath(dir_game)
self.dirs_container.setRootIndex(rt_index)
def get_selected_dir(self):
model = self.dirs_container.model()
ind = self.dirs_container.currentIndex()
file_path = model.filePath(ind)
if os.path.isdir(file_path):
return file_path
def handle_path(self, save_over=True):
# get path
if self.in_folder.isChecked():
root_dir = self.get_selected_dir()
if root_dir:
# walk path
ovls = walker.walk_type(root_dir, extension=".ovl")
for ovl_path in ovls:
# open ovl file
self.file_widget.decide_open(ovl_path)
# process each
yield self.ovl_data
if save_over:
self.ovl_data.save(ovl_path, "")
else:
interaction.showdialog("Select a root directory!")
# just the one that's currently open
else:
yield self.ovl_data
def dirs_clicked(self, ind):
# handle double clicked file paths
try:
file_path = ind.model().filePath(ind)
if os.path.isdir(file_path):
os.startfile(file_path)
elif file_path.lower().endswith(".ovl"):
self.file_widget.decide_open(file_path)
except BaseException as err:
print(err)
@staticmethod
def open_tools_dir():
os.startfile(os.getcwd())
def drag_files(self, file_names):
logging.info(f"DRAGGING {file_names}")
drag = QtGui.QDrag(self)
temp_dir = tempfile.mkdtemp("-cobra")
try:
out_paths, errors = self.ovl_data.extract(
temp_dir,
only_names=file_names,
show_temp_files=self.show_temp_files)
data = QtCore.QMimeData()
data.setUrls(
[QtCore.QUrl.fromLocalFile(path) for path in out_paths])
drag.setMimeData(data)
drag.exec_()
logging.info(
f"Tried to extract {len(file_names)} files, got {len(errors)} errors"
)
except BaseException as ex:
traceback.print_exc()
interaction.showdialog(str(ex))
logging.error(ex)
shutil.rmtree(temp_dir)
def rename_handle(self, old_name, new_name):
"""this manages the renaming of a single entry"""
names = [
(old_name, new_name),
]
self.ovl_data.rename(names)
self.update_gui_table()
def game_changed(self):
game = self.game_choice.entry.currentText()
# we must set both the context, and the local variable
set_game(self.ovl_data.context, game)
set_game(self.ovl_data, game)
def compression_changed(self):
compression = self.compression_choice.entry.currentText()
compression_value = Compression[compression]
self.ovl_data.context.user_version.compression = compression_value
self.ovl_data.user_version.compression = compression_value
@property
def commands(self):
# get those commands that are set to True
return [x for x in ("write_dat", ) if getattr(self, x)]
@property
def show_temp_files(self, ):
return self.t_show_temp_files.isChecked()
@property
def use_ext_dat(self, ):
return self.ext_dat.isChecked()
@property
def write_dat(self, ):
return self.t_write_dat.isChecked()
def dat_show(self, ):
if self.use_ext_dat:
self.dat_widget.show()
else:
self.dat_widget.hide()
def update_commands(self):
# at some point, just set commands to archive and trigger changes there
if self.file_widget.filename:
self.ovl_data.commands = self.commands
def update_progress(self, message, value=None, vmax=None):
# avoid gui updates if the value won't actually change the percentage.
# this saves us from making lots of GUI update calls that don't really
# matter.
try:
if vmax > 100 and (value % (vmax // 100)) and value != 0:
value = None
except ZeroDivisionError:
value = 0
except TypeError:
value = None
# update progress bar values if specified
if value is not None:
self.p_action.setValue(value)
if vmax is not None:
self.p_action.setMaximum(vmax)
# don't update the GUI unless the message has changed. label updates
# are expensive
if self.t_action_current_message != message:
self.t_action.setText(message)
self.t_action_current_message = message
def show_dependencies(self, file_index):
# just an example of what can be done when something is selected
file_entry = self.ovl_data.files[file_index]
ss_entry = self.ovl_data.get_sized_str_entry(file_entry.name)
ss_p = ss_entry.pointers[0]
logging.debug(
f"File: {ss_p.pool_index} {ss_p.data_offset} {ss_entry.name}")
for dep in file_entry.dependencies:
p = dep.pointers[0]
logging.debug(
f"Dependency: {p.pool_index} {p.data_offset} {dep.name}")
for f in ss_entry.fragments:
p0 = f.pointers[0]
p1 = f.pointers[1]
logging.debug(
f"Fragment: {p0.pool_index} {p0.data_offset} {p1.pool_index} {p1.data_offset}"
)
def load(self):
if self.file_widget.filepath:
self.file_widget.dirty = False
try:
# runTask(self.ovl_data.load, (self.file_widget.filepath,), {"commands": self.commands,})
# test(2)
# self.ovl_thread.func = self.ovl_thread.ovl_data.load
# self.ovl_thread.args = (self.file_widget.filepath,)
# self.ovl_thread.kwargs = {"commands": self.commands,}
# self.ovl_thread.start()
self.ovl_data.load(self.file_widget.filepath,
commands=self.commands)
# print(self.ovl_data.user_version)
# print(self.ovl_data)
# for a in self.ovl_data.archives:
# print(a)
# for a in self.ovl_data.archives[1:]:
# print(a.name)
# for ss in a.content.sized_str_entries:
# print(ss.name)
# print(self.ovl_data.mimes)
# print(self.ovl_data.triplets)
# for a, z in zip(self.ovl_data.archives, self.ovl_data.zlibs):
# print(a, z)
# print(f"zlib sum {z.zlib_thing_1 + z.zlib_thing_2 - 68}")
# print(f"pool size {a.pools_end - a.pools_start}")
# print(f"stream links size {12 * a.num_files}")
# print(f"buffer size {sum([buff.size for buff in a.content.buffer_entries])}")
# print(f"d1 size {sum([data.size_1 for data in a.content.data_entries])}")
# print(f"d2 size {sum([data.size_2 for data in a.content.data_entries])}")
# if a.name != "STATIC":
# streams = self.ovl_data.stream_files[a.stream_files_offset: a.stream_files_offset+a.num_files]
# print(a.name, streams)
# print(self.ovl_data.stream_files)
# for i, f in enumerate(self.ovl_data.files):
# if f.ext == ".texturestream":
# print(i, f.name)
# offsets = list(sorted((f.file_offset, i) for i, f in enumerate(self.ovl_data.stream_files)))
# # print(self.ovl_data)
# print(offsets)
# # for a in self.ovl_data.archives[1:]:
# # print(a.content)
# for sf in self.ovl_data.stream_files:
# print(sf)
# for a in self.ovl_data.archives:
# if a.pools_start <= sf.stream_offset < a.pools_end:
# print(f"is in {a.name}")
# print(f"pool offset relative {sf.stream_offset - a.pools_start}")
# # print(a.content.sized_str_entries)
# for a in self.ovl_data.archives:
# if a.name == "STATIC":
# for i, pool in enumerate(a.content.pools):
# if pool.offset <= sf.file_offset < pool.offset + pool.size:
# print(f"static pool {i} offset relative {sf.file_offset - pool.offset}")
# logging.debug(a.content)
# print(self.ovl_data.user_version)
except Exception as ex:
traceback.print_exc()
interaction.showdialog(str(ex))
print(self.ovl_data)
self.update_gui_table()
game = get_game(self.ovl_data)[0]
self.game_choice.entry.setText(game.value)
self.compression_choice.entry.setText(
self.ovl_data.user_version.compression.name)
def create_ovl(self, ovl_dir):
# clear the ovl
self.ovl_data = OvlFile(progress_callback=self.update_progress)
self.game_changed()
try:
self.ovl_data.create(ovl_dir)
except Exception as ex:
traceback.print_exc()
interaction.showdialog(str(ex))
self.update_gui_table()
def is_open_ovl(self):
if not self.file_widget.filename:
interaction.showdialog("You must open an OVL file first!")
else:
return True
def update_gui_table(self, ):
start_time = time.time()
logging.info(f"Loading {len(self.ovl_data.files)} files into gui")
self.files_container.set_data([[f.name, f.ext, f.file_hash]
for f in self.ovl_data.files])
self.included_ovls_view.set_data(self.ovl_data.included_ovl_names)
logging.info(f"Loaded GUI in {time.time() - start_time:.2f} seconds!")
self.update_progress("Operation completed!", value=1, vmax=1)
def save_as_ovl(self):
if self.is_open_ovl():
filepath = QtWidgets.QFileDialog.getSaveFileName(
self,
'Save OVL',
os.path.join(self.cfg.get("dir_ovls_out", "C://"),
self.file_widget.filename),
"OVL files (*.ovl)",
)[0]
if filepath:
self.cfg["dir_ovls_out"], ovl_name = os.path.split(filepath)
self._save_ovl(filepath)
def save_ovl(self):
if self.is_open_ovl():
self._save_ovl(self.file_widget.filepath)
def _save_ovl(self, filepath):
try:
ext_path = self.dat_widget.filepath if self.use_ext_dat else ""
self.ovl_data.save(filepath, ext_path)
self.file_widget.dirty = False
self.update_progress("Operation completed!", value=1, vmax=1)
except BaseException as ex:
traceback.print_exc()
interaction.showdialog(str(ex))
def extract_all(self):
out_dir = QtWidgets.QFileDialog.getExistingDirectory(
self,
'Output folder',
self.cfg.get("dir_extract", "C://"),
)
if out_dir:
self.cfg["dir_extract"] = out_dir
_out_dir = out_dir
all_error_files = []
for ovl in self.handle_path(save_over=False):
if self.is_open_ovl():
# for bulk extraction, add the ovl basename to the path to avoid overwriting
if self.in_folder.isChecked():
root_dir = self.get_selected_dir()
rel_p = os.path.relpath(ovl.path_no_ext,
start=root_dir)
out_dir = os.path.join(_out_dir, rel_p)
try:
out_paths, error_files = ovl.extract(
out_dir, show_temp_files=self.show_temp_files)
all_error_files += error_files
except Exception as ex:
traceback.print_exc()
interaction.showdialog(str(ex))
interaction.extract_error_warning(all_error_files)
def inject_ask(self):
if self.is_open_ovl():
files = QtWidgets.QFileDialog.getOpenFileNames(
self, 'Inject files', self.cfg.get("dir_inject", "C://"),
self.filter)[0]
self.inject_files(files)
def inject_files(self, files):
"""Tries to inject files into self.ovl_data"""
if files:
self.cfg["dir_inject"] = os.path.dirname(files[0])
try:
error_files, foreign_files = self.ovl_data.inject(
files, self.show_temp_files)
self.file_widget.dirty = True
if foreign_files:
if interaction.showdialog(
f"Do you want to add {len(foreign_files)} files to this ovl?",
ask=True):
self.ovl_data.add_files(foreign_files)
self.update_gui_table()
except Exception as ex:
traceback.print_exc()
interaction.showdialog(str(ex))
def get_replace_strings(self):
try:
newline = "\n"
old = self.e_name_old.toPlainText()
new = self.e_name_new.toPlainText()
old = old.split(newline)
new = new.split(newline)
if len(old) != len(new):
interaction.showdialog(
f"Old {len(old)} and new {len(new)} must have the same amount of lines!"
)
return list(zip(old, new))
except BaseException as err:
print(err)
def rename(self):
names = self.get_replace_strings()
if names:
for ovl in self.handle_path():
if self.is_open_ovl():
self.ovl_data.rename(
names, animal_mode=self.t_animal_ovl.isChecked())
self.update_gui_table()
def rename_contents(self):
names = self.get_replace_strings()
if names:
if self.check_length(names):
return
for ovl in self.handle_path():
if self.is_open_ovl():
self.ovl_data.rename_contents(names)
self.update_gui_table()
# Save the OVL file list to disk
def save_file_list(self):
if self.is_open_ovl():
filelist_src = QtWidgets.QFileDialog.getSaveFileName(
self,
'Save File List',
os.path.join(self.cfg.get("dir_ovls_out", "C://"),
self.file_widget.filename + ".files.txt"),
"Txt file (*.txt)",
)[0]
if filelist_src:
try:
file_names = self.files_container.table.get_files()
with open(filelist_src, 'w') as f:
f.write("\n".join(file_names))
self.update_progress("Operation completed!",
value=1,
vmax=1)
except BaseException as ex:
traceback.print_exc()
interaction.showdialog(str(ex))
# Save the OVL include list to disk
def save_included_ovls(self):
if self.is_open_ovl():
filelist_src = QtWidgets.QFileDialog.getSaveFileName(
self,
'ovls.include',
os.path.join(self.cfg.get("dir_ovls_out", "C://"),
"ovls.include"),
"Include file (*.include)",
)[0]
if filelist_src:
try:
self.ovl_data.save_included_ovls(filelist_src)
self.update_progress("Operation completed!",
value=1,
vmax=1)
except BaseException as ex:
traceback.print_exc()
interaction.showdialog(str(ex))
def remover(self):
if self.is_open_ovl():
selected_file_names = self.files_container.table.get_selected_files(
)
# todo - might want to check self.files_container.hasFocus(), but does not seem to work!
if selected_file_names:
try:
remover.file_remover(self.ovl_data, selected_file_names)
except:
traceback.print_exc()
self.update_gui_table()
def walker_hash(self, ):
start_dir = QtWidgets.QFileDialog.getExistingDirectory(
self, 'Game Root folder', self.cfg.get("dir_ovls_in", "C://"))
walker.generate_hash_table(self, start_dir)
self.update_progress("Operation completed!", value=1, vmax=1)
def walker_fgm(self, ):
start_dir = QtWidgets.QFileDialog.getExistingDirectory(
self, 'Game Root folder', self.cfg.get("dir_ovls_in", "C://"))
walker.get_fgm_values(self, start_dir)
self.update_progress("Operation completed!", value=1, vmax=1)
def inspect_models(self):
start_dir = QtWidgets.QFileDialog.getExistingDirectory(
self, 'Game Root folder', self.cfg.get("dir_ovls_in", "C://"))
walker.bulk_test_models(self, start_dir, walk_ovls=False)
self.update_progress("Operation completed!", value=1, vmax=1)
def closeEvent(self, event):
if self.file_widget.dirty:
quit_msg = f"Quit? You will lose unsaved work on {os.path.basename(self.file_widget.filepath)}!"
if not interaction.showdialog(quit_msg, ask=True):
event.ignore()
return
event.accept()
@staticmethod
def check_length(name_tups):
# Ask and return true if error is found and process should be stopped
for old, new in name_tups:
if len(old) != len(new):
if interaction.showdialog(
f"WARNING: length of '{old}' [{len(old)}] and '{new}' [{len(new)}] don't match!\n"
f"Stop renaming?",
ask=True):
return True
@staticmethod
def check_version():
is_64bits = sys.maxsize > 2**32
if not is_64bits:
interaction.showdialog(
"Either your operating system or your python installation is not 64 bits.\n"
"Large OVLs will crash unexpectedly!")
if sys.version_info[0] != 3 or sys.version_info[1] < 7 or (
sys.version_info[1] == 7 and sys.version_info[2] < 6):
interaction.showdialog("Python 3.7.6+ x64 bit is expected!")
class Mod():
def __init__(self, gui):
self.gui = gui
self.OVLs = []
global dir_path
dir_path = path.dirname(path.realpath(__file__)).replace("\\", "/")
def loadMeta(self, filepath):
with ZipFile(filepath) as zipfile:
with zipfile.open("mod.json") as jsonFile:
self.metaData = loads(jsonFile.read())
self.modName = (self.metaData["meta"]["Name"])
self.modAuthor = (self.metaData["meta"]["Author(s)"])
self.modDesc = (self.metaData["meta"]["Desc"])
def save(self):
with open(self.gui.modsJsonPath, "r") as file_r:
tempdata = file_r.read()
if len(tempdata) == 0:
self.modData = {}
else:
self.modData = loads(tempdata)
self.modData[self.modName] = {}
self.modData[self.modName] = ({
"Name": self.modName,
"Author": self.modAuthor,
"Desc": self.modDesc,
"Backups": self.backupPaths,
"OVLs": self.OVLs
})
print(self.modData)
with open(self.gui.modsJsonPath, "w") as file:
dump(self.modData, file)
def uninstall(self):
#self.gui.modList.pop(self.gui.modList.index(self))
for i, backup in enumerate(self.backupPaths):
self.backuppath = "{}/{}".format(self.gui.backupDir,
backup.replace("/", "]"))
#print(self.backuppath)
shutil.copyfile(self.backuppath,
self.gui.planetCoasterDir + "/" + self.OVLs[i])
if path.exists(self.backuppath[:-1] + "s"):
shutil.copyfile(
self.backuppath[:-1] + "s",
self.gui.planetCoasterDir + "/" + self.OVLs[i][:-1] + "s")
print("bonked {}".format(self.modName))
with open(self.gui.modsJsonPath, "r") as file_r:
tempdata = file_r.read()
if len(tempdata) == 0:
self.modData = {}
else:
self.modData = loads(tempdata)
self.modData.pop(self.modName, None)
with open(self.gui.modsJsonPath, "w") as file:
dump(self.modData, file)
self.gui.modList.pop(self.gui.modList.index(self))
def install(self, filepath):
with ZipFile(filepath) as zipfile:
with zipfile.open("mod.json") as jsonFile:
self.mod = load(jsonFile)
self.modName = (self.mod["meta"]["Name"])
self.modAuthor = (self.mod["meta"]["Author(s)"])
self.modDesc = (self.mod["meta"]["Desc"])
for self.path, self.files in self.mod["Files"].items():
print("{}/temp-files".format(self.gui.dataDir))
self.temppath = "{}/temp-files".format(self.gui.dataDir)
try:
mkdir(self.temppath)
except:
pass
self.backupPaths = []
self.backupPath = self.path[self.path.find("Win64"):]
self.backupPaths.append(self.backupPath)
self.backup(self.backupPath)
self.filesTemp = []
for self.file in self.files:
#Major sanitisation required
self.sanitised_path = self.path.replace("/", "_")
self.sanitised_path = self.sanitised_path.replace(
":", "#") #Needed?
self.sanitised_path = self.sanitised_path[
self.sanitised_path.find("Win64"):]
self.sanitised_file = self.file.split("\\")
self.name = self.sanitised_path + "/" + self.sanitised_file[
-1]
print("NAME: " + self.name)
print("TEMPPATH + " + self.temppath)
zipfile.extract(self.name, path=self.temppath)
self.fileSplit = self.file.rsplit("\\", 1)
self.file = "{}/{}/{}".format(self.temppath,
self.sanitised_path,
self.file)
print("File: " + self.file)
self.filesTemp.append(self.file)
self.ovlPath = self.gui.planetCoasterDir + "/" + self.path
self.OVLs.append(self.backupPath)
#self.save()
self.inject_files(self.ovlPath, self.filesTemp)
shutil.rmtree(self.temppath)
def inject_files(self, path, files):
self.ovl_data = OvlFile()
self.ovl_data.load(path)
inject.inject(self.ovl_data, files, False, False)
self.ovl_data.save(path)
def backup(self, relativeFilepath):
try:
mkdir(self.gui.backupDir)
except:
pass
self.directory = self.gui.planetCoasterDir + "/" + relativeFilepath
self.destination = self.getBackupPathDestination(relativeFilepath)
self.ovsDir = self.directory[:-1] + "s"
self.ovsDestination = self.getBackupPathDestination(
relativeFilepath)[:-1] + "s"
if path.exists(self.directory):
shutil.copyfile(self.directory, self.destination)
else:
pass
if path.exists(self.ovsDir):
shutil.copyfile(self.ovsDir, self.ovsDestination)
def getBackupPathDestination(self, relativeFilepath):
self.backupOVLPath = "{}\{}".format(
self.gui.backupDir,
relativeFilepath.replace("/", "]").replace("\\",
"]").replace(":", "#"))
return self.backupOVLPath
def walker(self, dummy=False, walk_ovls=True, walk_models=True):
start_dir = QtWidgets.QFileDialog.getExistingDirectory(
self,
'Game Root folder',
self.cfg.get("dir_ovls_in", "C://"),
)
errors = []
if start_dir:
export_dir = os.path.join(start_dir, "walker_export")
# don't use internal data
ovl_data = OvlFile()
mdl2_data = Mdl2File()
if walk_ovls:
error_files = []
skip_files = []
ovl_files = walker.walk_type(start_dir, extension="ovl")
of_max = len(ovl_files)
for of_index, ovl_path in enumerate(ovl_files):
self.update_progress("Walking OVL files: " +
os.path.basename(ovl_path),
value=of_index,
vmax=of_max)
try:
# read ovl file
ovl_data.load(ovl_path, commands=self.commands)
# create an output folder for it
outdir = os.path.join(export_dir,
os.path.basename(ovl_path[:-4]))
# create output dir
os.makedirs(outdir, exist_ok=True)
error_files_new, skip_files_new = extract.extract(
ovl_data.ovs_files[0],
outdir,
only_types=[
"ms2",
])
error_files += error_files_new
skip_files += skip_files_new
except Exception as ex:
traceback.print_exc()
errors.append((ovl_path, ex))
self.skip_messages(error_files, skip_files)
# holds different types of flag - list of byte maps pairs
type_dic = {}
if walk_models:
mdl2_files = walker.walk_type(export_dir, extension="mdl2")
mf_max = len(mdl2_files)
for mf_index, mdl2_path in enumerate(mdl2_files):
mdl2_name = os.path.basename(mdl2_path)
self.update_progress("Walking MDL2 files: " + mdl2_name,
value=mf_index,
vmax=mf_max)
try:
mdl2_data.load(mdl2_path, quick=True, map_bytes=True)
for model in mdl2_data.models:
if model.flag not in type_dic:
type_dic[model.flag] = ([], [])
type_dic[model.flag][0].append(mdl2_name)
type_dic[model.flag][1].append(model.bytes_map)
except Exception as ex:
traceback.print_exc()
errors.append((mdl2_path, ex))
# report
print("\nThe following errors occured:")
for file_path, ex in errors:
print(file_path, str(ex))
print("\nThe following type - map pairs were found:")
for flag, tup in sorted(type_dic.items()):
print(flag, bin(flag))
names, maps_list = tup
print("Some files:", list(set(names))[:25])
print("num models", len(maps_list))
print("mean",
np.mean(maps_list, axis=0).astype(dtype=np.ubyte))
print("max", np.max(maps_list, axis=0))
print()
self.update_progress("Operation completed!", value=1, vmax=1)
class MainWindow(widgets.MainWindow):
def __init__(self):
widgets.MainWindow.__init__(
self,
"OVL Tool",
)
self.resize(720, 400)
self.ovl_data = OvlFile(progress_callback=self.update_progress)
supported_types = ("DDS", "PNG", "MDL2", "TXT", "FGM", "FDB", "MATCOL",
"XMLCONFIG", "ASSETPKG", "LUA", "WEM", "OTF", "TTF")
self.filter = "Supported files ({})".format(" ".join(
"*." + t for t in supported_types))
self.file_widget = widgets.FileWidget(self, self.cfg)
self.file_widget.setToolTip(
"The name of the OVL file that is currently open.")
self.p_action = QtWidgets.QProgressBar(self)
self.p_action.setGeometry(0, 0, 200, 15)
self.p_action.setTextVisible(True)
self.p_action.setMaximum(1)
self.p_action.setValue(0)
self.t_action_current_message = "No operation in progress"
self.t_action = QtWidgets.QLabel(self,
text=self.t_action_current_message)
# header_names = ["Name", "File Type", "Size", "Compressed Size", "DJB", "Fragments"]
header_names = ["Name", "File Type", "DJB", "Unk0", "Unk1"]
self.table = widgets.TableView(header_names, self)
# toggles
self.t_show_temp_files = QtWidgets.QCheckBox("Save Temp Files")
self.t_show_temp_files.setToolTip(
"By default, temporary files are converted to usable ones and back on the fly."
)
self.t_show_temp_files.setChecked(False)
self.t_2K = QtWidgets.QCheckBox("Inject 2K")
self.t_2K.setToolTip(
"Experimental: Increase a JWE Diffuse or Normal map to 2048x2048 resolution."
)
self.t_2K.setChecked(False)
self.t_write_dat = QtWidgets.QCheckBox("Save DAT")
self.t_write_dat.setToolTip(
"Writes decompressed archive streams to DAT files for debugging.")
self.t_write_dat.setChecked(False)
self.t_write_dat.stateChanged.connect(self.load)
self.t_write_frag_log = QtWidgets.QCheckBox("Save Frag Log")
self.t_write_frag_log.setToolTip("For devs.")
self.t_write_frag_log.setChecked(False)
self.t_write_frag_log.stateChanged.connect(self.load)
self.qgrid = QtWidgets.QGridLayout()
self.qgrid.addWidget(self.file_widget, 0, 0, 1, 4)
self.qgrid.addWidget(self.t_show_temp_files, 1, 0)
self.qgrid.addWidget(self.t_write_dat, 1, 1)
self.qgrid.addWidget(self.t_write_frag_log, 1, 2)
self.qgrid.addWidget(self.t_2K, 1, 3)
self.qgrid.addWidget(self.table, 2, 0, 1, 4)
self.qgrid.addWidget(self.p_action, 3, 0, 1, 4)
self.qgrid.addWidget(self.t_action, 4, 0, 1, 4)
self.central_widget.setLayout(self.qgrid)
mainMenu = self.menuBar()
fileMenu = mainMenu.addMenu('File')
editMenu = mainMenu.addMenu('Edit')
helpMenu = mainMenu.addMenu('Help')
button_data = (
(fileMenu, "Open", self.file_widget.ask_open, "CTRL+O", "dir"),
(fileMenu, "Save", self.save_ovl, "CTRL+S", "save"),
(fileMenu, "Exit", self.close, "", "exit"),
(editMenu, "Unpack", self.extract_all, "CTRL+U", "extract"),
(editMenu, "Inject", self.inject, "CTRL+I", "inject"),
# (editMenu, "Hash", self.hasher, "CTRL+H", ""),
(editMenu, "Walk", self.walker, "", ""),
(editMenu, "Generate Hash Table", self.walker_hash, "", ""),
(helpMenu, "Report Bug", self.report_bug, "", "report"),
(helpMenu, "Documentation", self.online_support, "", "manual"))
self.add_to_menu(button_data)
self.check_version()
self.load_hash_table()
@property
def commands(self, ):
# get those commands that are set to True
return [x for x in ("write_dat", "write_frag_log") if getattr(self, x)]
@property
def show_temp_files(self, ):
return self.t_show_temp_files.isChecked()
@property
def write_2K(self, ):
return self.t_2K.isChecked()
@property
def write_dat(self, ):
return self.t_write_dat.isChecked()
@property
def write_frag_log(self, ):
return self.t_write_frag_log.isChecked()
def update_commands(self):
# at some point, just set commands to archive and trigger changes there
if self.file_widget.filename:
self.ovl_data.commands = self.commands
def update_progress(self, message, value=None, vmax=None):
# avoid gui updates if the value won't actually change the percentage.
# this saves us from making lots of GUI update calls that don't really
# matter.
try:
if vmax > 100 and (value % (vmax // 100)) and value != 0:
value = None
except ZeroDivisionError:
value = 0
except TypeError:
value = None
# update progress bar values if specified
if value is not None:
self.p_action.setValue(value)
if vmax is not None:
self.p_action.setMaximum(vmax)
# don't update the GUI unless the message has changed. label updates
# are expensive
if self.t_action_current_message != message:
self.t_action.setText(message)
self.t_action_current_message = message
def load_hash_table(self):
print("Loading hash table...")
start_time = time.time()
self.hash_table = {}
hashes_dir = os.path.join(os.getcwd(), "hashes")
try:
for file in os.listdir(hashes_dir):
if file.endswith(".txt"):
with open(os.path.join(hashes_dir, file), "r") as f:
for line in f:
line = line.strip()
if line:
k, v = line.split(" = ")
self.hash_table[int(k)] = v
except:
pass
# print(self.hash_table)
print(
f"Loaded {len(self.hash_table)} hash - name pairs in {time.time()-start_time:.2f} seconds."
)
def load(self):
if self.file_widget.filepath:
self.file_widget.dirty = False
start_time = time.time()
self.update_progress("Reading OVL " + self.file_widget.filepath,
value=0,
vmax=0)
try:
self.ovl_data.load(self.file_widget.filepath,
commands=self.commands,
hash_table=self.hash_table)
except Exception as ex:
traceback.print_exc()
widgets.showdialog(str(ex))
print(ex)
data = []
# dic = {}
print(f"Loading {len(self.ovl_data.files)} files into gui...")
for file_w in self.ovl_data.files:
name = f"{file_w.name}.{file_w.ext}"
line = [
name, file_w.ext,
to_hex_str(file_w.file_hash),
str(file_w.unkn_0),
str(file_w.unkn_1)
]
data.append(line)
# dic[file_w.file_hash] = name
# print(dic)
# print(self.ovl_data)
print("loading gui")
self.table.set_data(data)
print(f"Done in {time.time()-start_time:.2f} seconds!")
self.update_progress("Operation completed!", value=1, vmax=1)
def save_ovl(self):
if self.file_widget.filename:
file_src = QtWidgets.QFileDialog.getSaveFileName(
self,
'Save OVL',
os.path.join(self.cfg.get("dir_ovls_out", "C://"),
self.file_widget.filename),
"OVL files (*.ovl)",
)[0]
if file_src:
self.cfg["dir_ovls_out"], ovl_name = os.path.split(file_src)
try:
self.ovl_data.save(file_src)
except BaseException as error:
print(error)
self.file_widget.dirty = False
print("Done!")
def skip_messages(self, error_files, skip_files):
error_count = len(error_files)
skip_count = len(skip_files)
if error_count:
print("Files not extracted due to error:")
for ef in error_files:
print("\t", ef)
if skip_count:
print("Unsupported files not extracted:")
for sf in skip_files:
print("\t", sf)
if error_count or skip_count:
message = f"{error_count + skip_count} files were not extracted from the archive and may be missing from the output folder. {skip_count} were unsupported, while {error_count} produced errors."
widgets.showdialog(message)
def extract_all(self):
if self.file_widget.filename:
out_dir = QtWidgets.QFileDialog.getExistingDirectory(
self,
'Output folder',
self.cfg.get("dir_extract", "C://"),
)
if out_dir:
self.cfg["dir_extract"] = out_dir
# create output dir
try:
os.makedirs(out_dir, exist_ok=True)
archive = self.ovl_data.ovs_files[0]
error_files, skip_files = extract.extract(
archive,
out_dir,
self.show_temp_files,
progress_callback=self.update_progress)
self.skip_messages(error_files, skip_files)
self.update_progress("Operation completed!",
value=1,
vmax=1)
except Exception as ex:
traceback.print_exc()
widgets.showdialog(str(ex))
print(ex)
else:
widgets.showdialog(
"You must open an OVL file before you can extract files!")
def inject(self):
if self.file_widget.filename:
files = QtWidgets.QFileDialog.getOpenFileNames(
self, 'Inject files', self.cfg.get("dir_inject", "C://"),
self.filter)[0]
if files:
self.cfg["dir_inject"] = os.path.dirname(files[0])
try:
inject.inject(self.ovl_data, files, self.show_temp_files,
self.write_2K)
self.file_widget.dirty = True
except Exception as ex:
traceback.print_exc()
widgets.showdialog(str(ex))
print("Done!")
else:
widgets.showdialog(
"You must open an OVL file before you can inject files!")
def hasher(self):
if self.file_widget.filename:
names = [(tup[0].text(), tup[1].text())
for tup in self.e_name_pairs]
for archive in self.ovl_data.archives:
hasher.dat_hasher(archive, names, self.ovl_data.header.files,
self.ovl_data.header.textures)
else:
widgets.showdialog(
"You must open an OVL file before you can extract files!")
def walker_hash(self, dummy=False, walk_ovls=True, walk_models=True):
start_dir = QtWidgets.QFileDialog.getExistingDirectory(
self,
'Game Root folder',
self.cfg.get("dir_ovls_in", "C://"),
)
hash_dict = {}
if start_dir:
# don't use internal data
ovl_data = OvlFile()
error_files = []
ovl_files = walker.walk_type(start_dir, extension="ovl")
of_max = len(ovl_files)
for of_index, ovl_path in enumerate(ovl_files):
self.update_progress("Hashing names: " +
os.path.basename(ovl_path),
value=of_index,
vmax=of_max)
try:
# read ovl file
new_hashes = ovl_data.load(
ovl_path, commands=("generate_hash_table", ))
hash_dict.update(new_hashes)
except:
error_files.append(ovl_path)
if error_files:
print(f"{error_files} caused errors!")
# write the hash text file to the hashes folder
export_dir = os.path.join(os.getcwd(), "hashes")
out_path = os.path.join(export_dir,
f"{os.path.basename(start_dir)}.txt")
with open(out_path, "w") as f:
for k, v in hash_dict.items():
f.write(f"{k} = {v}\n")
def walker(self, dummy=False, walk_ovls=True, walk_models=True):
start_dir = QtWidgets.QFileDialog.getExistingDirectory(
self,
'Game Root folder',
self.cfg.get("dir_ovls_in", "C://"),
)
errors = []
if start_dir:
export_dir = os.path.join(start_dir, "walker_export")
# don't use internal data
ovl_data = OvlFile()
mdl2_data = Mdl2File()
if walk_ovls:
error_files = []
skip_files = []
ovl_files = walker.walk_type(start_dir, extension="ovl")
of_max = len(ovl_files)
for of_index, ovl_path in enumerate(ovl_files):
self.update_progress("Walking OVL files: " +
os.path.basename(ovl_path),
value=of_index,
vmax=of_max)
try:
# read ovl file
ovl_data.load(ovl_path, commands=self.commands)
# create an output folder for it
outdir = os.path.join(export_dir,
os.path.basename(ovl_path[:-4]))
# create output dir
os.makedirs(outdir, exist_ok=True)
error_files_new, skip_files_new = extract.extract(
ovl_data.ovs_files[0],
outdir,
only_types=[
"ms2",
])
error_files += error_files_new
skip_files += skip_files_new
except Exception as ex:
traceback.print_exc()
errors.append((ovl_path, ex))
self.skip_messages(error_files, skip_files)
# holds different types of flag - list of byte maps pairs
type_dic = {}
if walk_models:
mdl2_files = walker.walk_type(export_dir, extension="mdl2")
mf_max = len(mdl2_files)
for mf_index, mdl2_path in enumerate(mdl2_files):
mdl2_name = os.path.basename(mdl2_path)
self.update_progress("Walking MDL2 files: " + mdl2_name,
value=mf_index,
vmax=mf_max)
try:
mdl2_data.load(mdl2_path, quick=True, map_bytes=True)
for model in mdl2_data.models:
if model.flag not in type_dic:
type_dic[model.flag] = ([], [])
type_dic[model.flag][0].append(mdl2_name)
type_dic[model.flag][1].append(model.bytes_map)
except Exception as ex:
traceback.print_exc()
errors.append((mdl2_path, ex))
# report
print("\nThe following errors occured:")
for file_path, ex in errors:
print(file_path, str(ex))
print("\nThe following type - map pairs were found:")
for flag, tup in sorted(type_dic.items()):
print(flag, bin(flag))
names, maps_list = tup
print("Some files:", list(set(names))[:25])
print("num models", len(maps_list))
print("mean",
np.mean(maps_list, axis=0).astype(dtype=np.ubyte))
print("max", np.max(maps_list, axis=0))
print()
self.update_progress("Operation completed!", value=1, vmax=1)
def closeEvent(self, event):
if self.file_widget.dirty:
qm = QtWidgets.QMessageBox
quit_msg = "You will lose unsaved work on " + os.path.basename(
self.file_widget.filepath) + "!"
reply = qm.question(self, 'Quit?', quit_msg, qm.Yes, qm.No)
if reply == qm.Yes:
event.accept()
else:
event.ignore()
else:
event.accept()
@staticmethod
def check_version():
is_64bits = sys.maxsize > 2**32
if not is_64bits:
widgets.showdialog(
"Either your operating system or your python installation is not 64 bits.\n"
"Large OVLs will crash unexpectedly!")
if sys.version_info[0] != 3 or sys.version_info[1] < 7 or (
sys.version_info[1] == 7 and sys.version_info[2] < 6):
widgets.showdialog("Python 3.7.6+ x64 bit is expected!")