def mods_from_arc(request, tmpdir_factory):
blender = pytest.config.getoption('blender')
if not blender:
pytest.skip('No blender bin path supplied')
import_arc_filepath = request.param
if import_arc_filepath.endswith(tuple(KNOWN_ARC_BLENDER_CRASH)):
pytest.xfail('Known arc crashes blender')
log_filepath = str(tmpdir_factory.getbasetemp().join('blender.log'))
import_unpack_dir = TemporaryDirectory()
export_arc_filepath = os.path.join(gettempdir(), os.path.basename(import_arc_filepath))
script_filepath = os.path.join(gettempdir(), 'import_arc.py')
with open(script_filepath, 'w') as w:
w.write(PYTHON_TEMPLATE.format(project_dir=os.getcwd(),
import_arc_filepath=import_arc_filepath,
export_arc_filepath=export_arc_filepath,
import_unpack_dir=import_unpack_dir.name,
log_filepath=log_filepath))
args = '{} -noaudio --background --python {}'.format(blender, script_filepath)
try:
subprocess.check_output((args,), shell=True)
except subprocess.CalledProcessError:
# the test will actually error here, if the import/export fails, since the file it won't exist.
# which is better, since pytest traceback to subprocess.check_output is pretty long and useless
with open(log_filepath) as f:
for line in f:
print(line) # XXX: should print only the last n lines
os.unlink(export_arc_filepath)
os.unlink(script_filepath)
raise
export_unpack_dir = TemporaryDirectory()
arc = Arc(export_arc_filepath)
arc.unpack(export_unpack_dir.name)
mod_files_original = [os.path.join(root, f) for root, _, files in os.walk(import_unpack_dir.name)
for f in files if f.endswith('.mod')]
mod_files_exported = [os.path.join(root, f) for root, _, files in os.walk(export_unpack_dir.name)
for f in files if f.endswith('.mod')]
mod_files_original = sorted(mod_files_original, key=os.path.basename)
mod_files_exported = sorted(mod_files_exported, key=os.path.basename)
mod_objects_original = []
mod_objects_exported = []
if mod_files_original and mod_files_exported:
os.unlink(export_arc_filepath)
os.unlink(script_filepath)
for i, mod_file_original in enumerate(mod_files_original):
mod_original = Mod156(file_path=mod_file_original)
mod_exported = Mod156(file_path=mod_files_exported[i])
mod_objects_original.append(mod_original)
mod_objects_exported.append(mod_exported)
else:
os.unlink(export_arc_filepath)
os.unlink(script_filepath)
pytest.skip('Arc contains no mod files')
return mod_objects_original, mod_objects_exported
def test_arc_from_dir_re5(tmpdir, arc_file):
"""get an arc file (ideally from the game), unpack it, repackit, unpack it again
compare the 2 arc files and the 2 output folders"""
arc_original = Arc(file_path=arc_file)
arc_original_out = os.path.join(str(tmpdir), os.path.basename(arc_file).replace('.arc', ''))
arc_original.unpack(arc_original_out)
arc_from_dir = Arc.from_dir(arc_original_out)
arc_from_dir_out = os.path.join(str(tmpdir), 'arc-from-dir.arc')
with open(arc_from_dir_out, 'wb') as w:
w.write(arc_from_dir)
arc_from_arc_from_dir = Arc(file_path=arc_from_dir_out)
arc_from_arc_from_dir_out = os.path.join(str(tmpdir), 'arc-from-arc-from-dir')
arc_from_arc_from_dir.unpack(arc_from_arc_from_dir_out)
files_extracted_1 = [f for _, _, files in os.walk(arc_original_out) for f in files]
files_extracted_2 = [f for _, _, files in os.walk(arc_from_arc_from_dir_out) for f in files]
# Assumming zlib default compression used in all original arc files.
assert os.path.getsize(arc_file) == os.path.getsize(arc_from_dir_out)
# The hashes would be different due to the file_paths ordering
assert arc_original.files_count == arc_from_arc_from_dir.files_count
assert sorted(files_extracted_1) == sorted(files_extracted_2)
assert arc_from_arc_from_dir.file_entries[0].offset == 32768
def mods_from_arc(request, tmpdir_factory):
arc_file = request.param
base_temp = tmpdir_factory.mktemp(os.path.basename(arc_file).replace('.arc', '-arc'))
out = str(base_temp)
arc = Arc(file_path=arc_file)
arc.unpack(out)
mod_files = [os.path.join(root, f) for root, _, files in os.walk(out)
for f in files if f.endswith('.mod')]
mods = [Mod156(mod_file) for mod_file in mod_files]
for i, mod in enumerate(mods):
assert ctypes.sizeof(mod) == os.path.getsize(mod_files[i])
return mods
def test_arc_unpack_re5(tmpdir, arc_file):
arc = Arc(file_path=arc_file)
out = os.path.join(str(tmpdir), 'extracted_arc')
arc.unpack(out)
files = {os.path.join(root, f) for root, _, files in os.walk(out)
for f in files}
expected_sizes = sorted([f.size for f in arc.file_entries if f.size])
files_sizes = sorted([os.path.getsize(f) for f in files])
assert os.path.isdir(out)
assert arc.files_count == len(files)
assert expected_sizes == files_sizes
def import_arc(file_path, extraction_dir=None, context_scene=None):
'''Imports an arc file (Resident Evil 5 for only for now) into blender,
extracting all files to a tmp dir and saving unknown/unused data
to the armature (if any) for using in exporting'''
if file_path.endswith(
tuple(KNOWN_ARC_BLENDER_CRASH) + tuple(CORRUPTED_ARCS)):
raise ValueError(
'The arc file provided is not supported yet, it might crash Blender'
)
base_dir = os.path.basename(file_path).replace('.arc', '_arc_extracted')
out = extraction_dir or os.path.join(os.path.expanduser('~'), '.albam',
're5', base_dir)
if not os.path.isdir(out):
os.makedirs(out)
if not out.endswith(os.path.sep):
out = out + os.path.sep
arc = Arc(file_path=file_path)
arc.unpack(out)
arc_name = os.path.basename(file_path)
mod_files = [
os.path.join(root, f) for root, _, files in os.walk(out) for f in files
if f.endswith('.mod')
]
mod_folders = [
os.path.dirname(mod_file.split(out)[-1]) for mod_file in mod_files
]
# Saving arc to main object
parent = bpy.data.objects.new(arc_name, None)
bpy.context.scene.objects.link(parent)
parent.albam_imported_item['data'] = bytes(arc)
parent.albam_imported_item.name = arc_name
parent.albam_imported_item.source_path = file_path
parent.albam_imported_item.file_type = 'mtframework.arc'
for i, mod_file in enumerate(mod_files):
import_mod(mod_file, out, parent, mod_folders[i])
# Addding the name of the imported item so then it can be selected
# from a list for exporting. Exporting models without a base model,
# at least for models with skeleton doesn't make much sense, plus
# the arc files contain a lot of other files that are not imported, but
# saved in the blend file
new_albam_imported_item = context_scene.albam_items_imported.add()
new_albam_imported_item.name = os.path.basename(file_path)
def import_arc(file_path, extraction_dir=None, context_scene=None):
'''Imports an arc file (Resident Evil 5 for only for now) into blender,
extracting all files to a tmp dir and saving unknown/unused data
to the armature (if any) for using in exporting'''
if file_path.endswith(tuple(KNOWN_ARC_BLENDER_CRASH) + tuple(CORRUPTED_ARCS)):
raise ValueError('The arc file provided is not supported yet, it might crash Blender')
base_dir = os.path.basename(file_path).replace('.arc', '_arc_extracted')
out = extraction_dir or os.path.join(os.path.expanduser('~'), '.albam', 're5', base_dir)
if not os.path.isdir(out):
os.makedirs(out)
if not out.endswith(os.path.sep):
out = out + os.path.sep
arc = Arc(file_path=file_path)
arc.unpack(out)
arc_name = os.path.basename(file_path)
mod_files = [os.path.join(root, f) for root, _, files in os.walk(out)
for f in files if f.endswith('.mod')]
mod_folders = [os.path.dirname(mod_file.split(out)[-1]) for mod_file in mod_files]
# Saving arc to main object
parent = bpy.data.objects.new(arc_name, None)
bpy.context.scene.objects.link(parent)
parent.albam_imported_item['data'] = bytes(arc)
parent.albam_imported_item.name = arc_name
parent.albam_imported_item.source_path = file_path
parent.albam_imported_item.file_type = 'mtframework.arc'
for i, mod_file in enumerate(mod_files):
import_mod(mod_file, out, parent, mod_folders[i])
# Addding the name of the imported item so then it can be selected
# from a list for exporting. Exporting models without a base model,
# at least for models with skeleton doesn't make much sense, plus
# the arc files contain a lot of other files that are not imported, but
# saved in the blend file
new_albam_imported_item = context_scene.albam_items_imported.add()
new_albam_imported_item.name = os.path.basename(file_path)
def export_arc(blender_object):
'''Exports an arc file containing mod and tex files, among others from a
previously imported arc.'''
mods = {}
try:
saved_arc = Arc(
file_path=BytesIO(blender_object.albam_imported_item.data))
except AttributeError:
raise ExportError(
'Object {0} did not come from the original arc'.format(
blender_object.name))
for child in blender_object.children:
try:
basename = posixpath.basename(child.name)
folder = child.albam_imported_item.folder
if os.sep == ntpath.sep: # Windows
mod_filepath = ntpath.join(ensure_ntpath(folder), basename)
else:
mod_filepath = os.path.join(folder, basename)
except AttributeError:
raise ExportError(
'Object {0} did not come from the original arc'.format(
child.name))
assert child.albam_imported_item.file_type == 'mtframework.mod'
mod, textures = export_mod156(child)
mods[mod_filepath] = (mod, textures)
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir_slash_ending = tmpdir + os.sep if not tmpdir.endswith(
os.sep) else tmpdir
saved_arc.unpack(tmpdir)
mod_files = [
os.path.join(root, f) for root, _, files in os.walk(tmpdir)
for f in files if f.endswith('.mod')
]
# tex_files = {os.path.join(root, f) for root, _, files in os.walk(tmpdir)
# for f in files if f.endswith('.tex')}
new_tex_files = set()
for modf in mod_files:
rel_path = modf.split(tmpdir_slash_ending)[1]
try:
new_mod = mods[rel_path]
except KeyError:
raise ExportError(
"Can't export to arc, a mod file is missing: {}".format(
rel_path))
with open(modf, 'wb') as w:
w.write(new_mod[0])
mod_textures = new_mod[1]
for texture in mod_textures:
tex = Tex112.from_dds(
file_path=bpy.path.abspath(texture.image.filepath))
try:
tex.unk_float_1 = texture.albam_imported_texture_value_1
tex.unk_float_2 = texture.albam_imported_texture_value_2
tex.unk_float_3 = texture.albam_imported_texture_value_3
tex.unk_float_4 = texture.albam_imported_texture_value_4
except AttributeError:
pass
tex_name = os.path.basename(texture.image.filepath)
tex_filepath = os.path.join(os.path.dirname(modf),
tex_name.replace('.dds', '.tex'))
new_tex_files.add(tex_filepath)
with open(tex_filepath, 'wb') as w:
w.write(tex)
# probably other files can reference textures besides mod, this is in case
# textures applied have other names.
# TODO: delete only textures referenced from saved_mods at import time
# unused_tex_files = tex_files - new_tex_files
# for utex in unused_tex_files:
# os.unlink(utex)
new_arc = Arc.from_dir(tmpdir)
return new_arc
def export_arc(blender_object):
'''Exports an arc file containing mod and tex files, among others from a
previously imported arc.'''
mods = {}
try:
saved_arc = Arc(file_path=BytesIO(blender_object.albam_imported_item.data))
except AttributeError:
raise ExportError('Object {0} did not come from the original arc'.format(blender_object.name))
for child in blender_object.children:
try:
basename = posixpath.basename(child.name)
folder = child.albam_imported_item.folder
if os.sep == ntpath.sep: # Windows
mod_filepath = ntpath.join(ensure_ntpath(folder), basename)
else:
mod_filepath = os.path.join(folder, basename)
except AttributeError:
raise ExportError('Object {0} did not come from the original arc'.format(child.name))
assert child.albam_imported_item.file_type == 'mtframework.mod'
mod, textures = export_mod156(child)
mods[mod_filepath] = (mod, textures)
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir_slash_ending = tmpdir + os.sep if not tmpdir.endswith(os.sep) else tmpdir
saved_arc.unpack(tmpdir)
mod_files = [os.path.join(root, f) for root, _, files in os.walk(tmpdir)
for f in files if f.endswith('.mod')]
# tex_files = {os.path.join(root, f) for root, _, files in os.walk(tmpdir)
# for f in files if f.endswith('.tex')}
new_tex_files = set()
for modf in mod_files:
rel_path = modf.split(tmpdir_slash_ending)[1]
try:
new_mod = mods[rel_path]
except KeyError:
raise ExportError("Can't export to arc, a mod file is missing: {}".format(rel_path))
with open(modf, 'wb') as w:
w.write(new_mod[0])
mod_textures = new_mod[1]
for texture in mod_textures:
tex = Tex112.from_dds(file_path=bpy.path.abspath(texture.image.filepath))
try:
tex.unk_float_1 = texture.albam_imported_texture_value_1
tex.unk_float_2 = texture.albam_imported_texture_value_2
tex.unk_float_3 = texture.albam_imported_texture_value_3
tex.unk_float_4 = texture.albam_imported_texture_value_4
except AttributeError:
pass
tex_name = os.path.basename(texture.image.filepath)
tex_filepath = os.path.join(os.path.dirname(modf), tex_name.replace('.dds', '.tex'))
new_tex_files.add(tex_filepath)
with open(tex_filepath, 'wb') as w:
w.write(tex)
# probably other files can reference textures besides mod, this is in case
# textures applied have other names.
# TODO: delete only textures referenced from saved_mods at import time
# unused_tex_files = tex_files - new_tex_files
# for utex in unused_tex_files:
# os.unlink(utex)
new_arc = Arc.from_dir(tmpdir)
return new_arc
def mods_from_arc(request, tmpdir_factory):
blender = pytest.config.getoption('blender')
if not blender:
pytest.skip('No blender bin path supplied')
import_arc_filepath = request.param
if import_arc_filepath.endswith(tuple(KNOWN_ARC_BLENDER_CRASH)):
pytest.xfail('Known arc crashes blender')
log_filepath = str(tmpdir_factory.getbasetemp().join('blender.log'))
import_unpack_dir = TemporaryDirectory()
export_arc_filepath = os.path.join(gettempdir(),
os.path.basename(import_arc_filepath))
script_filepath = os.path.join(gettempdir(), 'import_arc.py')
with open(script_filepath, 'w') as w:
w.write(
PYTHON_TEMPLATE.format(project_dir=os.getcwd(),
import_arc_filepath=import_arc_filepath,
export_arc_filepath=export_arc_filepath,
import_unpack_dir=import_unpack_dir.name,
log_filepath=log_filepath))
args = '{} -noaudio --background --python {}'.format(
blender, script_filepath)
try:
subprocess.check_output((args, ), shell=True)
except subprocess.CalledProcessError:
# the test will actually error here, if the import/export fails, since the file it won't exist.
# which is better, since pytest traceback to subprocess.check_output is pretty long and useless
with open(log_filepath) as f:
for line in f:
print(line) # XXX: should print only the last n lines
os.unlink(export_arc_filepath)
os.unlink(script_filepath)
raise
export_unpack_dir = TemporaryDirectory()
arc = Arc(export_arc_filepath)
arc.unpack(export_unpack_dir.name)
mod_files_original = [
os.path.join(root, f)
for root, _, files in os.walk(import_unpack_dir.name) for f in files
if f.endswith('.mod')
]
mod_files_exported = [
os.path.join(root, f)
for root, _, files in os.walk(export_unpack_dir.name) for f in files
if f.endswith('.mod')
]
mod_files_original = sorted(mod_files_original, key=os.path.basename)
mod_files_exported = sorted(mod_files_exported, key=os.path.basename)
mod_objects_original = []
mod_objects_exported = []
if mod_files_original and mod_files_exported:
os.unlink(export_arc_filepath)
os.unlink(script_filepath)
for i, mod_file_original in enumerate(mod_files_original):
mod_original = Mod156(file_path=mod_file_original)
mod_exported = Mod156(file_path=mod_files_exported[i])
mod_objects_original.append(mod_original)
mod_objects_exported.append(mod_exported)
else:
os.unlink(export_arc_filepath)
os.unlink(script_filepath)
pytest.skip('Arc contains no mod files')
return mod_objects_original, mod_objects_exported