def do_tag_extract(self):
tag_path = self.tag_path.get()
def_id = self.tag_class_ext_to_fcc.get(
Path(tag_path).suffix[1:].lower())
if def_id is None or def_id not in self.tag_data_extractors:
print("Cannot extract data from this kind of tag.")
return
else:
if not is_in_dir(tag_path, self.handler.tagsdir):
print("Tag %s is not located within tags directory: %s" %
(tag_path, self.handler.tagsdir))
return
print("Extracting %s" % tag_path)
self.extract(os.path.relpath(tag_path, str(self.handler.tagsdir)),
self.tag_data_extractors[def_id],
out_dir=str(
Path(path_split(self.handler.tagsdir,
"tags")).joinpath("data")),
overwrite=self.overwrite.get(),
engine="yelo",
decode_adpcm=self.decode_adpcm.get())
print("Extraction completed.\n")
def populate_dependency_tree(self):
filepath = self.tag_filepath.get()
if not filepath:
return
app = self.app_root
handler = self.handler = app.handler
handler_name = app.handler_names[app._curr_handler_index]
if handler_name not in app.tags_dir_relative:
print("Change the current tag set.")
return
filepath = path_normalize(filepath)
if not is_in_dir(filepath, self.handler.tagsdir):
print("%s\nis not in tagsdir\n%s" %
(filepath, self.handler.tagsdir))
return
rel_filepath = Path(filepath).relative_to(self.handler.tagsdir)
tag = self.get_tag(rel_filepath)
if tag is None:
print("Could not load tag:\n %s" % filepath)
return
self.dependency_frame.handler = handler
self.dependency_frame.tags_dir = self.handler.tagsdir
self.dependency_frame.root_tag_path = tag.filepath
self.dependency_frame.root_tag_text = rel_filepath
self.dependency_frame.reload()
def tag_browse(self):
if self._extracting:
return
filetypes = [('All', '*')]
for def_id in sorted(self.tag_data_extractors.keys()):
if def_id in self.tag_class_fcc_to_ext:
filetypes.append(
(def_id, "." + self.tag_class_fcc_to_ext[def_id]))
fp = askopenfilename(initialdir=str(self.app_root.last_load_dir),
filetypes=filetypes,
parent=self,
title="Select a tag to extract from")
if not fp:
return
fp = Path(fp)
self.app_root.last_load_dir = fp.parent
if not is_in_dir(fp, self.handler.tagsdir):
print("Tag %s is not located in tags directory %s" %
(fp, self.handler.tagsdir))
return
self.app_root.last_load_dir = fp.parent
self.tag_path.set(fp)
def browse_shader_path(self):
if self._compiling or self._loading or self._saving:
return
tags_dir = self.tags_dir.get()
if not tags_dir or not os.path.exists(tags_dir):
return
shader_dir = os.path.dirname(
os.path.join(tags_dir, self.shader_path_string_var.get()))
shader_exts = tuple((typ, "*.shader_%s" % typ) for typ in shader_types)
fp = asksaveasfilename(
initialdir=shader_dir,
parent=self,
title="Select the shader to use(or where to make one)",
filetypes=shader_exts + (('All', '*'), ))
fp, ext = os.path.splitext(fp)
if fp:
if not is_in_dir(fp, tags_dir):
print("Specified shader is not located in the tags directory.")
return
ext = ext.strip(".").lower()
self.shader_path_string_var.set(os.path.relpath(fp, tags_dir))
mat = self.get_material(self.shader_names_menu.sel_index)
if mat and ext in shader_type_map:
self.shader_types_menu.sel_index = shader_type_map[ext]
mat.shader_type = ext
def jms_dir_browse(self):
if self._compiling or self._loading or self._saving:
return
tags_dir = self.tags_dir.get()
data_dir = path_replace(tags_dir, "tags", "data")
jms_dir = self.jms_dir.get()
if tags_dir and not jms_dir:
jms_dir = data_dir
dirpath = askdirectory(
initialdir=jms_dir,
parent=self,
title="Select the folder of models to compile...")
if not dirpath:
return
dirpath = str(Path(dirpath))
if tags_dir and data_dir and os.path.basename(
dirpath).lower() == "models":
object_dir = os.path.dirname(dirpath)
if object_dir and is_in_dir(object_dir, data_dir):
tag_path = os.path.join(object_dir,
os.path.basename(object_dir))
tag_path = os.path.join(tags_dir,
os.path.relpath(tag_path, data_dir))
self.gbxmodel_path.set(tag_path + ".gbxmodel")
self.app_root.last_load_dir = os.path.dirname(dirpath)
self.jms_dir.set(dirpath)
if not self.tags_dir.get():
self.tags_dir.set(
Path(path_split(self.app_root.last_load_dir, "data"), "tags"))
def get_backup_dir(self, filepath=None):
filepath = Path(os.path.realpath(str(filepath)))
fallback_dir = filepath.parent.joinpath(self.backup_dir_basename)
if not self.tagsdir_relative:
return fallback_dir
# TODO: Test that is_in_dir works here
if not is_in_dir(filepath, self.tagsdir):
return fallback_dir
return self.tagsdir.joinpath(
self.backup_dir_basename,
os.path.dirname(os.path.relpath(str(filepath), str(self.tagsdir))))
def dir_browse(self):
if self._scanning:
return
dirpath = askdirectory(initialdir=self.directory_path.get(),
parent=self,
title="Select directory to scan")
if not dirpath:
return
self.app_root.last_load_dir = dirpath
if not is_in_dir(dirpath, self.handler.tagsdir):
print(
"Specified directory is not located within the tags directory")
return
self.directory_path.set(dirpath)
def dir_browse(self):
if self._extracting:
return
dirpath = askdirectory(
initialdir=self.dir_path.get(),
parent=self,
title="Select the directory of tags to extract from")
if not dirpath:
return
self.app_root.last_load_dir = dirpath
if not is_in_dir(dirpath, self.handler.tagsdir):
print('Directory "%s" is not located inside tags dir: "%s"' %
(dirpath, self.handler.tagsdir))
return
self.dir_path.set(dirpath)
def get_next_backup_filepath(self, filepath, backup_count=1):
filepath = Path(filepath)
backup_count = max(backup_count, 1)
backup_dir = self.get_backup_dir(filepath)
existing_backup_paths = self.get_backup_paths_by_timestamps(filepath)
if existing_backup_paths and len(
existing_backup_paths) >= backup_count:
return existing_backup_paths[sorted(existing_backup_paths)[0]]
if self.tagsdir_relative and is_in_dir(filepath, self.tagsdir):
backup_path = self.tagsdir.joinpath(
self.backup_dir_basename,
os.path.relpath(str(filepath), str(self.tagsdir)))
else:
backup_path = backup_dir.joinpath(filepath.stem)
return self.get_unique_filename(backup_path,
set(existing_backup_paths.values()),
())
def jma_dir_browse(self):
if self._compiling or self._loading or self._saving:
return
tags_dir = self.tags_dir.get()
# Add data to the path and then use path_replace to match the case of any
# data directory that might already be here.
data_dir = str(
path_replace(
Path(tags_dir).parent.joinpath("data"), "data", "data"))
jma_dir = self.jma_dir.get()
if tags_dir and not jma_dir:
jma_dir = data_dir
dirpath = path_normalize(
askdirectory(
initialdir=jma_dir,
parent=self,
title="Select the folder of animations to compile..."))
if not dirpath:
return
dirpath = str(Path(dirpath))
if tags_dir and data_dir and os.path.basename(
dirpath).lower() == "animations":
object_dir = os.path.dirname(dirpath)
if object_dir and is_in_dir(object_dir, data_dir):
tag_path = os.path.join(object_dir,
os.path.basename(object_dir))
tag_path = os.path.join(tags_dir,
os.path.relpath(tag_path, data_dir))
self.model_animations_path.set(tag_path + ".model_animations")
self.app_root.last_load_dir = os.path.dirname(dirpath)
self.jma_dir.set(dirpath)
if not self.tags_dir.get():
self.tags_dir.set(
os.path.join(path_split(self.app_root.last_load_dir, "data"),
"tags"))
def tags_dir_browse(self):
if self._compiling or self._loading or self._saving:
return
old_tags_dir = self.tags_dir.get()
tags_dir = askdirectory(initialdir=old_tags_dir,
parent=self,
title="Select the root of the tags directory")
if not tags_dir:
return
tags_dir = str(Path(tags_dir))
antr_path = self.model_animations_path.get()
if old_tags_dir and antr_path and not is_in_dir(antr_path, tags_dir):
# adjust antr filepath to be relative to the new tags directory
antr_path = os.path.join(tags_dir,
os.path.relpath(antr_path, old_tags_dir))
self.model_animations_path.set(antr_path)
self.app_root.last_load_dir = os.path.dirname(tags_dir)
self.tags_dir.set(tags_dir)
def tags_dir_browse(self):
if self._compiling or self._loading or self._saving:
return
old_tags_dir = self.tags_dir.get()
tags_dir = askdirectory(initialdir=old_tags_dir,
parent=self,
title="Select the root of the tags directory")
if not tags_dir:
return
tags_dir = path_normalize(tags_dir)
mod2_path = self.gbxmodel_path.get()
if old_tags_dir and mod2_path and not is_in_dir(mod2_path, tags_dir):
# adjust mod2 filepath to be relative to the new tags directory
mod2_path = os.path.join(tags_dir,
os.path.relpath(mod2_path, old_tags_dir))
self.gbxmodel_path.set(mod2_path)
self.app_root.last_load_dir = os.path.dirname(tags_dir)
self.tags_dir.set(tags_dir)
def do_recursive_zip(self):
tag_path = self.tag_filepath.get()
if not tag_path:
return
app = self.app_root
handler = self.handler = app.handler
handler_name = app.handler_names[app._curr_handler_index]
if handler_name not in app.tags_dir_relative:
print("Change the current tag set.")
return
tag_path = Path(tag_path)
if not is_in_dir(tag_path, self.handler.tagsdir):
print("Specified tag is not located within the tags directory")
return
tagzip_path = asksaveasfilename(initialdir=self.app_root.last_load_dir,
parent=self,
title="Save zipfile to...",
filetypes=(("zipfile", "*.zip"), ))
if not tagzip_path:
return
try:
rel_filepath = tag_path.relative_to(self.handler.tagsdir)
tag = self.get_tag(rel_filepath)
except ValueError:
tag = None
if tag is None:
print("Could not load tag:\n %s" % tag_path)
return
# make the zipfile to put everything in
tagzip_path = os.path.splitext(tagzip_path)[0] + ".zip"
tags_to_zip = [rel_filepath]
new_tags_to_zip = []
seen_tags = set()
with zipfile.ZipFile(str(tagzip_path), mode='w') as tagzip:
# loop over all the tags and add them to the zipfile
while tags_to_zip:
for rel_filepath in tags_to_zip:
tag_path = tagpath_to_fullpath(
self.handler.tagsdir, PureWindowsPath(rel_filepath))
if self.stop_zipping:
print('Recursive zip operation cancelled.\n')
return
if rel_filepath in seen_tags:
continue
seen_tags.add(rel_filepath)
try:
print("Adding '%s' to zipfile" % rel_filepath)
app.update_idletasks()
tag = self.get_tag(rel_filepath)
new_tags_to_zip.extend(self.get_dependencies(tag))
# try to conserve memory a bit
del tag
tagzip.write(str(tag_path), arcname=str(rel_filepath))
except Exception:
print(format_exc())
print(" Could not add '%s' to zipfile." %
rel_filepath)
# replace the tags to zip with the newly collected ones
tags_to_zip[:] = new_tags_to_zip
del new_tags_to_zip[:]
print("\nRecursive zip completed.\n")
def do_dir_extract(self):
tags_path = self.dir_path.get()
data_path = self.handler.tagsdir.parent.joinpath("data")
if not is_in_dir(tags_path, self.handler.tagsdir):
print("Directory %s is not located inside tags dir: %s" %
(tags_path, self.handler.tagsdir))
return
settings = dict(out_dir=str(data_path),
overwrite=self.overwrite.get(),
decode_adpcm=self.decode_adpcm.get(),
engine="yelo")
print("Beginning tag data extracton in:\t%s" % self.handler.tagsdir)
s_time = time()
c_time = s_time
p_int = self.print_interval
all_tag_paths = {
self.listbox_index_to_def_id[int(i)]: []
for i in self.def_ids_listbox.curselection()
}
print("Locating tags...")
for root, directories, files in os.walk(tags_path):
root = Path(root)
try:
root = root.relative_to(self.handler.tagsdir)
except ValueError:
continue
for filename in files:
filepath = root.joinpath(filename)
if time() - c_time > p_int:
c_time = time()
print(' ' * 4, filepath, sep="")
self.app_root.update_idletasks()
if self.stop_extracting:
print('Tag data extraction cancelled.\n')
return
tag_paths = all_tag_paths.get(
self.tag_class_ext_to_fcc.get(filepath.suffix[1:].lower()))
if tag_paths is not None:
tag_paths.append(filepath)
for def_id in sorted(all_tag_paths):
extractor = self.tag_data_extractors[def_id]
if self.stop_extracting:
print('Tag data extraction cancelled.\n')
return
print("Extracting %s" % def_id)
for filepath in all_tag_paths[def_id]:
if self.stop_extracting:
print('Tag data extraction cancelled.\n')
return
print(' ' * 4, filepath, sep="")
self.extract(filepath, extractor, **settings)
print("Extraction completed.\n")
def scan(self):
handler = self.handler
self.stop_scanning = False
logpath = path_normalize(self.logfile_path.get())
dirpath = path_normalize(self.directory_path.get())
if not is_in_dir(dirpath, self.handler.tagsdir):
print(
"Specified directory is not located within the tags directory")
return
#this is the string to store the entire debug log
log_name = "HEK Tag Scanner log"
debuglog = "\n%s%s%s\n\n" % ("-" * 30, log_name, "-" *
(50 - len(log_name)))
debuglog += "tags directory = %s\nscan directory = %s\n\n" % (
self.handler.tagsdir, dirpath)
debuglog += "Broken dependencies are listed below.\n"
tag_specific_errors = {}
get_nodes = handler.get_nodes_by_paths
get_tagref_invalid = handler.get_tagref_invalid
s_time = time()
c_time = s_time
p_int = self.print_interval
all_tag_paths = {
self.listbox_index_to_def_id[int(i)]: []
for i in self.def_ids_listbox.curselection()
}
ext_id_map = handler.ext_id_map
id_ext_map = handler.id_ext_map
print("Locating tags...")
for root, directories, files in os.walk(dirpath):
root = path_normalize(os.path.join(root, ""))
rel_root = Path(root).relative_to(self.handler.tagsdir)
for filename in files:
filepath = rel_root.joinpath(filename)
if time() - c_time > p_int:
c_time = time()
print(' ' * 4, filepath, sep="")
self.app_root.update_idletasks()
if self.stop_scanning:
print('Tag scanning operation cancelled.\n')
return
tag_paths = all_tag_paths.get(
ext_id_map.get(os.path.splitext(filename)[-1].lower()))
if tag_paths is not None:
tag_paths.append(filepath)
# make the debug string by scanning the tags directory
for def_id in sorted(all_tag_paths.keys()):
tag_ref_paths = handler.tag_ref_cache.get(def_id)
self.app_root.update_idletasks()
print("Scanning '%s' tags..." % id_ext_map[def_id][1:])
tags_coll = all_tag_paths[def_id]
# always display the first tag's filepath
c_time = time() - (p_int + 100)
for filepath in sorted(tags_coll):
if self.stop_scanning:
print('Tag scanning operation cancelled.\n')
break
if time() - c_time > p_int:
c_time = time()
print(' ' * 4, filepath, sep="")
self.app_root.update_idletasks()
tag = self.get_tag(self.handler.tagsdir.joinpath(filepath))
if tag is None:
print(" Could not load '%s'" % filepath)
continue
# find tag specific errors
self.tag_specific_scan(tag, tag_specific_errors)
try:
if tag_ref_paths is None:
# no dependencies for this tag. continue on
continue
missed = get_nodes(tag_ref_paths, tag.data,
get_tagref_invalid)
if not missed:
continue
debuglog += "\n\n%s\n" % filepath
block_name = None
for block in missed:
if block.NAME != block_name:
debuglog += '%s%s\n' % (' ' * 4, block.NAME)
block_name = block.NAME
try:
ext = '.' + block.tag_class.enum_name
except Exception:
ext = ''
debuglog += '%s%s\n' % (' ' * 8, block.STEPTREE + ext)
except Exception:
print(format_exc())
print(" Could not scan '%s'" % tag.filepath)
continue
if self.stop_scanning:
break
if tag_specific_errors:
debuglog += "\nTag specific errors are listed below.\n"
for def_id in sorted(tag_specific_errors.keys()):
debuglog += "\n\n%s specific errors:\n%s" % (
def_id, tag_specific_errors[def_id])
print("\nScanning took %s seconds." % int(time() - s_time))
print("Writing logfile to %s..." % logpath)
self.app_root.update_idletasks()
# make and write to the logfile
try:
handler.make_log_file(debuglog, logpath)
try:
print("Scan completed.\n")
if self.open_logfile.get():
open_in_default_program(logpath)
except Exception:
print("Could not open written log.")
return
except Exception:
print("Could not create log. Printing log to console instead.\n\n")
for line in debuglog.split('\n'):
try:
print(line)
except Exception:
print("<COULD NOT PRINT THIS LINE>")
print("Scan completed.\n")
def _load_models(self):
models_dir = self.jms_dir.get()
if not models_dir:
return
start = time.time()
print("Locating jms files...")
fps = []
for _, __, files in os.walk(models_dir):
for fname in files:
ext = os.path.splitext(fname)[-1].lower()
#if ext in ".jms.obj.dae":
if ext in ".jms.obj":
fps.append(os.path.join(models_dir, fname))
break
if not fps:
print(" No valid jms files found in the folder.")
return
self.mod2_tag = self.merged_jms = None
optimize_level = max(0, self.optimize_menu.sel_index)
jms_models = self.jms_models = []
print("Loading jms files...")
self.app_root.update()
for fp in fps:
try:
print(" %s" % fp.replace('/', '\\').split("\\")[-1])
self.app_root.update()
model_name = os.path.basename(fp).split('.')[0]
ext = os.path.splitext(fp)[-1].lower()
jms_model = None
if ext == ".jms":
with open(fp, "r") as f:
jms_model = read_jms(f.read(), '', model_name)
elif ext == ".obj":
with open(fp, "r") as f:
jms_model = jms_model_from_obj(f.read(), model_name)
elif ext == ".dae":
jms_model = jms_model_from_dae(fp, model_name)
if not jms_model:
continue
jms_models.append(jms_model)
if optimize_level:
old_vert_ct = len(jms_model.verts)
print(" Optimizing...", end='')
jms_model.optimize_geometry(optimize_level == 1)
print(" Removed %s verts" %
(old_vert_ct - len(jms_model.verts)))
print(" Calculating normals...")
jms_model.calculate_vertex_normals()
except Exception:
print(format_exc())
print(" Could not parse jms file.")
self.app_root.update()
if not jms_models:
print(" No valid jms files found.")
return
first_crc = None
for jms_model in jms_models:
if first_crc is None:
first_crc = jms_model.node_list_checksum
elif first_crc != jms_model.node_list_checksum:
print(" Warning, not all node list checksums match.")
break
# make sure the highest lod for each permutation is set as superhigh
# this is necessary, as only superhigh jms markers are used
jms_models_by_name = {}
for jms_model in jms_models:
lod_models = jms_models_by_name.setdefault(jms_model.perm_name,
[None] * 5)
lod_index = {
"high": 1,
"medium": 2,
"low": 3,
"superlow": 4
}.get(jms_model.lod_level, 0)
lod_models[lod_index] = jms_model
for lod_models in jms_models_by_name.values():
for jms_model in lod_models:
if jms_model is not None:
jms_model.lod_level = "superhigh"
break
print("Merging jms data...")
self.app_root.update()
self.merged_jms = merged_jms = MergedJmsModel()
errors_occurred = False
for jms_model in jms_models:
errors = merged_jms.merge_jms_model(jms_model)
errors_occurred |= bool(errors)
if errors:
print(" Errors in '%s'" % jms_model.name)
for error in errors:
print(" ", error, sep='')
self.app_root.update()
mod2_path = self.gbxmodel_path.get()
tags_dir = self.tags_dir.get().replace('/', '\\')
self.shader_names_menu.max_index = len(merged_jms.materials) - 1
shaders_dir = ""
if mod2_path:
shaders_dir = os.path.join(os.path.dirname(mod2_path), "shaders",
'')
tags_dir = self.tags_dir.get()
has_local_shaders = os.path.exists(shaders_dir) and os.path.exists(
tags_dir)
if errors_occurred:
print(" Errors occurred while loading jms files.")
elif os.path.isfile(mod2_path):
try:
self.mod2_tag = mod2_def.build(filepath=mod2_path)
tagdata = self.mod2_tag.data.tagdata
self.merged_jms.node_list_checksum = tagdata.node_list_checksum
self.superhigh_lod_cutoff.set(str(
tagdata.superhigh_lod_cutoff))
self.high_lod_cutoff.set(str(tagdata.high_lod_cutoff))
self.medium_lod_cutoff.set(str(tagdata.medium_lod_cutoff))
self.low_lod_cutoff.set(str(tagdata.low_lod_cutoff))
self.superlow_lod_cutoff.set(str(tagdata.superlow_lod_cutoff))
# get any shaders in the gbxmodel and set the shader_path
# and shader_type for any matching materials in the jms
shdr_refs = {}
for shdr_ref in tagdata.shaders.STEPTREE:
shdr_name = shdr_ref.shader.filepath.split(
"\\")[-1].lower()
shdr_refs.setdefault(shdr_name, []).append(shdr_ref)
for mat in merged_jms.materials:
shdr_ref = shdr_refs.get(mat.name, [""]).pop(0)
if shdr_ref:
mat.shader_type = shdr_ref.shader.tag_class.enum_name
mat.shader_path = shdr_ref.shader.filepath
local_shaders = {}
if has_local_shaders and is_in_dir(shaders_dir, tags_dir):
# fill in any missing shader paths with ones found nearby
for _, __, files in os.walk(shaders_dir):
for filename in files:
name, ext = os.path.splitext(filename)
ext = ext.lower()
if ext.startswith(".shader"):
local_shaders.setdefault(
name.split("\\")[-1].lower(), []).append(
os.path.join(shaders_dir, filename))
break
for mat in merged_jms.materials:
shader_path = local_shaders.get(mat.name, [""]).pop(0)
if "shader_" in mat.shader_type or not shader_path:
continue
# shader type isnt set. Try to detect its location and
# type if possible, or set it to a default value if not
shader_path = shader_path.lower().replace("/", "\\")
name, ext = os.path.splitext(shader_path)
mat.shader_path = os.path.relpath(name,
tags_dir).strip("\\")
mat.shader_type = ext.strip(".")
except Exception:
print(format_exc())
else:
self.superhigh_lod_cutoff.set("0.0")
self.high_lod_cutoff.set("0.0")
self.medium_lod_cutoff.set("0.0")
self.low_lod_cutoff.set("0.0")
self.superlow_lod_cutoff.set("0.0")
for mat in merged_jms.materials:
shader_path = mat.shader_path
if mat.shader_type in ("shader", ""):
assume_shaders_dir = not shaders_dir
if not assume_shaders_dir:
try:
shader_path = os.path.relpath(
os.path.join(shaders_dir, shader_path), tags_dir)
shader_path = shader_path.strip("\\")
except ValueError:
assume_shaders_dir = True
mat.shader_type = "shader_model"
else:
assume_shaders_dir = False
if assume_shaders_dir or shader_path.startswith("..\\"):
shader_path = "shaders\\" + os.path.basename(shader_path)
mat.shader_path = shader_path.lstrip("..\\")
if not self.mod2_tag:
print(
" Existing gbxmodel tag not detected or could not be loaded.\n"
" A new gbxmodel tag will be created.")
print("Finished loading models. Took %.6f seconds.\n" %
(time.time() - start))
self.select_shader(0)
