def test_strategise_relative_only(self):
infile = self.blendfiles / 'absolute_path.blend'
packer = pack.Packer(infile,
self.blendfiles,
self.tpath,
relative_only=True)
packer.strategise()
packed_files = (
'absolute_path.blend',
# Linked with a relative path:
'textures/Bricks/brick_dotted_04-color.jpg',
# This file links to textures/Textures/Buildings/buildings_roof_04-color.png,
# but using an absolute path, so that file should be skipped.
)
for pf in packed_files:
path = self.blendfiles / pf
act = packer._actions[path]
self.assertEqual(pack.PathAction.KEEP_PATH, act.path_action,
'for %s' % pf)
self.assertEqual(self.tpath / pf, act.new_path, 'for %s' % pf)
self.assertEqual(len(packed_files), len(packer._actions))
self.assertEqual({}, self.rewrites(packer))
def test_missing_files(self):
cb = mock.Mock(progress.Callback)
infile = self.blendfiles / 'missing_textures.blend'
with pack.Packer(infile, self.blendfiles, self.tpath) as packer:
packer.progress_cb = cb
packer.strategise()
packer.execute()
self.assertEqual(1, cb.pack_start.call_count)
self.assertEqual(1, cb.pack_done.call_count)
cb.rewrite_blendfile.assert_not_called()
cb.transfer_file.assert_has_calls([
mock.call(infile, self.tpath / 'missing_textures.blend'),
mock.call(mock.ANY, self.tpath / 'pack-info.txt'),
],
any_order=True)
self.assertEqual(0, cb.transfer_file_skipped.call_count)
self.assertGreaterEqual(
cb.transfer_progress.call_count, 1,
'transfer_progress() should be called at least once per asset')
expected_calls = [
mock.call(
self.blendfiles /
'textures/HDRI/Myanmar/Golden Palace 2, Old Bagan-1k.exr'),
mock.call(self.blendfiles /
'textures/Textures/Marble/marble_decoration-color.png'),
]
cb.missing_file.assert_has_calls(expected_calls, any_order=True)
self.assertEqual(len(expected_calls), cb.missing_file.call_count)
def test_compression(self):
blendname = 'subdir/doubly_linked_up.blend'
imgfile = self.blendfiles / blendname
packer = pack.Packer(imgfile,
self.blendfiles,
self.tpath,
compress=True)
packer.strategise()
packer.execute()
dest = self.tpath / blendname
self.assertTrue(dest.exists())
self.assertTrue(blendfile.open_cached(dest).is_compressed)
for bpath in self.tpath.rglob('*.blend'):
if bpath == dest:
# Only test files that were bundled as dependency; the main
# file was tested above already.
continue
self.assertTrue(
blendfile.open_cached(bpath).is_compressed,
'Expected %s to be compressed' % bpath)
break
else:
self.fail('Expected to have Blend files in the BAT pack.')
for imgpath in self.tpath.rglob('*.jpg'):
with imgpath.open('rb') as imgfile:
magic = imgfile.read(3)
self.assertEqual(b'\xFF\xD8\xFF', magic,
'Expected %s to NOT be compressed' % imgpath)
break
else:
self.fail('Expected to have JPEG files in the BAT pack.')
def test_output_path(self):
infile = self.blendfiles / 'basic_file.blend'
packer = pack.Packer(infile, self.blendfiles.parent, self.tpath)
packer.strategise()
self.assertEqual(self.tpath / self.blendfiles.name / infile.name,
packer.output_path)
def test_exclude_filter(self):
# Files shouldn't be reported missing if they should be ignored.
infile = self.blendfiles / 'image_sequencer.blend'
with pack.Packer(infile, self.blendfiles, self.tpath) as packer:
packer.exclude('*.png', '*.nonsense')
packer.strategise()
packer.execute()
self.assertFalse((self.tpath / 'imgseq').exists())
def _pack_with_rewrite(self):
ppath = self.blendfiles / 'subdir'
infile = ppath / 'doubly_linked_up.blend'
packer = pack.Packer(infile, ppath, self.tpath)
packer.strategise()
packer.execute()
return infile, packer
def create_packer(args, bpath: pathlib.Path, ppath: pathlib.Path,
target: str) -> pack.Packer:
if target.startswith('s3:/'):
if args.noop:
raise ValueError('S3 uploader does not support no-op.')
if args.compress:
raise ValueError(
'S3 uploader does not support on-the-fly compression')
if args.relative_only:
raise ValueError(
'S3 uploader does not support the --relative-only option')
packer = create_s3packer(bpath, ppath, pathlib.PurePosixPath(target))
elif target.startswith('shaman+http:/') or target.startswith('shaman+https:/') \
or target.startswith('shaman:/'):
if args.noop:
raise ValueError('Shaman uploader does not support no-op.')
if args.compress:
raise ValueError(
'Shaman uploader does not support on-the-fly compression')
if args.relative_only:
raise ValueError(
'Shaman uploader does not support the --relative-only option')
packer = create_shamanpacker(bpath, ppath, target)
elif target.lower().endswith('.zip'):
from blender_asset_tracer.pack import zipped
if args.compress:
raise ValueError(
'ZIP packer does not support on-the-fly compression')
packer = zipped.ZipPacker(bpath,
ppath,
target,
noop=args.noop,
relative_only=args.relative_only)
else:
packer = pack.Packer(bpath,
ppath,
target,
noop=args.noop,
compress=args.compress,
relative_only=args.relative_only)
if args.exclude:
# args.exclude is a list, due to nargs='*', so we have to split and flatten.
globs = [glob for globs in args.exclude for glob in globs.split()]
log.info('Excluding: %s', ', '.join(repr(g) for g in globs))
packer.exclude(*globs)
return packer
def test_exclude_filter_missing_files(self):
# Files shouldn't be reported missing if they should be ignored.
infile = self.blendfiles / 'missing_textures.blend'
with pack.Packer(infile, self.blendfiles, self.tpath) as packer:
packer.exclude('*.png')
packer.strategise()
self.assertEqual([
self.blendfiles /
'textures/HDRI/Myanmar/Golden Palace 2, Old Bagan-1k.exr'
], list(packer.missing_files))
def test_missing_files(self):
infile = self.blendfiles / 'missing_textures.blend'
packer = pack.Packer(infile, self.blendfiles, self.tpath)
packer.strategise()
self.assertEqual([
self.blendfiles /
'textures/HDRI/Myanmar/Golden Palace 2, Old Bagan-1k.exr',
self.blendfiles /
'textures/Textures/Marble/marble_decoration-color.png'
], sorted(packer.missing_files))
def test_infofile(self):
blendname = 'subdir/doubly_linked_up.blend'
infile = self.blendfiles / blendname
packer = pack.Packer(infile, self.blendfiles, self.tpath)
packer.strategise()
packer.execute()
infopath = self.tpath / 'pack-info.txt'
self.assertTrue(infopath.exists())
info = infopath.open().read().splitlines(keepends=False)
self.assertEqual(blendname, info[-1].strip())
def pack_blend_file(blend_file_path, target_dir_path, progress_cb=None):
""" Packs a Blender .blend file to a target folder using blender_asset_tracer
:param blend_file_path: The .blend file to pack
:type blend_file_path: str
:param target_dir_path: The path to the directory which will contain the packed files
:type target_dir_path: str
:param progress_cb: The progress reporting callback instance
:type progress_cb: blender_asset_tracer.pack.progress.Callback
"""
log = get_wrapped_logger(__name__ + '.' + inspect.stack()[0][3])
log.info("Starting pack operation....")
blend_file_path = pathlib.Path(blend_file_path)
project_root_path = blend_file_path.parent
target_dir_path = pathlib.Path(target_dir_path)
log.info("blend_file_path: %s" % str(blend_file_path))
log.info("project_root_path: %s" % str(project_root_path))
log.info("target_dir_path: %s" % str(target_dir_path))
# create packer
with pack.Packer(
blend_file_path,
project_root_path,
target_dir_path,
noop=False, # no-op mode, only shows what will be done
compress=False,
relative_only=False) as packer:
log.info("Created packer")
if progress_cb:
log.info("Setting packer progress callback...")
packer._progress_cb = progress_cb
# plan the packing operation (must be called before execute)
log.info("Plan packing operation...")
packer.strategise()
# attempt to pack the project
try:
log.info("Plan packing operation...")
packer.execute()
except pack.transfer.FileTransferError as ex:
log.warning(
str(len(ex.files_remaining)) +
" files couldn't be copied, starting with " +
str(ex.files_remaining[0]))
raise SystemExit(1)
finally:
log.info("Exiting packing operation...")
def test_abort_strategise(self):
infile = self.blendfiles / 'subdir/doubly_linked_up.blend'
packer = pack.Packer(infile, self.blendfiles, self.tpath)
class AbortingCallback(progress.Callback):
def trace_blendfile(self, filename: pathlib.Path):
# Call abort() somewhere during the strategise() call.
if filename.name == 'linked_cube.blend':
packer.abort()
packer.progress_cb = AbortingCallback()
with packer, self.assertRaises(pack.Aborted):
packer.strategise()
def pack_scene(tpath: str):
bpy.ops.file.report_missing_files()
bpath = pathlib.Path(bpy.data.filepath)
print("Packing", bpath, "into", tpath)
bpath = bpathlib.make_absolute(bpath)
ppath = bpathlib.make_absolute(bpath).parent
packer = pack.Packer(bpath, ppath, tpath)
packer.strategise()
for missing_file in packer.missing_files:
notif('Warning - Missing file ' + str(missing_file))
packer.execute()
def test_abort_transfer(self):
infile = self.blendfiles / 'subdir/doubly_linked_up.blend'
packer = pack.Packer(infile, self.blendfiles, self.tpath)
first_file_size = infile.stat().st_size
class AbortingCallback(progress.Callback):
def transfer_progress(self, total_bytes: int,
transferred_bytes: int):
# Call abort() somewhere during the file transfer.
if total_bytes > first_file_size * 1.1:
packer.abort()
packer.progress_cb = AbortingCallback()
with packer:
packer.strategise()
with self.assertRaises(pack.Aborted):
packer.execute()
def test_noop(self):
ppath = self.blendfiles / 'subdir'
infile = ppath / 'doubly_linked_up.blend'
packer = pack.Packer(infile, ppath, self.tpath, noop=True)
packer.strategise()
packer.execute()
self.assertEqual([], list(self.tpath.iterdir()))
# The original file shouldn't be touched.
bfile = blendfile.open_cached(infile)
libs = sorted(bfile.code_index[b'LI'])
self.assertEqual(b'LILib', libs[0].id_name)
self.assertEqual(b'//../linked_cube.blend', libs[0][b'name'])
self.assertEqual(b'LILib.002', libs[1].id_name)
self.assertEqual(b'//../material_textures.blend', libs[1][b'name'])
def test_smoke_cache(self):
# The smoke cache uses a glob to indicate which files to pack.
cb = mock.Mock(progress.Callback)
infile = self.blendfiles / 'T55542-smoke/smoke_cache.blend'
with pack.Packer(infile, self.blendfiles, self.tpath) as packer:
packer.progress_cb = cb
packer.strategise()
packer.execute()
# We should have all the *.bphys files now.
count = len(
list((self.tpath /
'T55542-smoke/blendcache_smoke_cache').glob('*.bphys')))
self.assertEqual(10, count)
# Physics files + smoke_cache.blend + pack_info.txt
self.assertGreaterEqual(
cb.transfer_progress.call_count, 12,
'transfer_progress() should be called at least once per asset')
def test_particle_cache_with_ignore_glob(self):
cb = mock.Mock(progress.Callback)
infile = self.blendfiles / 'T55539-particles/particle.blend'
with pack.Packer(infile, self.blendfiles, self.tpath) as packer:
packer.progress_cb = cb
packer.exclude('*.bphys')
packer.strategise()
packer.execute()
# We should have none of the *.bphys files now.
count = len(
list((self.tpath /
'T55539-particles/blendcache_particle').glob('*.bphys')))
self.assertEqual(0, count)
# Just particle.blend + pack_info.txt
self.assertGreaterEqual(
cb.transfer_progress.call_count, 2,
'transfer_progress() should be called at least once per asset')
def test_execute_with_rewrite(self):
cb = mock.Mock(progress.Callback)
infile = self.blendfiles / 'subdir/doubly_linked_up.blend'
with pack.Packer(infile, infile.parent, self.tpath) as packer:
packer.progress_cb = cb
packer.strategise()
packer.execute()
self.assertEqual(1, cb.pack_start.call_count)
self.assertEqual(1, cb.pack_done.call_count)
# rewrite_blendfile should only be called paths in a blendfile are
# actually rewritten.
cb.rewrite_blendfile.assert_called_with(
self.blendfiles / 'subdir/doubly_linked_up.blend')
self.assertEqual(1, cb.rewrite_blendfile.call_count)
# mock.ANY is used for temporary files in temporary paths, because they
# are hard to predict.
extpath = self.outside_project()
expected_calls = [
mock.call(mock.ANY, self.tpath / 'doubly_linked_up.blend'),
mock.call(mock.ANY, self.tpath / 'pack-info.txt'),
mock.call(mock.ANY, extpath / 'linked_cube.blend'),
mock.call(mock.ANY, extpath / 'basic_file.blend'),
mock.call(mock.ANY, extpath / 'material_textures.blend'),
mock.call(
self.blendfiles / 'textures/Bricks/brick_dotted_04-color.jpg',
extpath / 'textures/Bricks/brick_dotted_04-color.jpg'),
mock.call(
self.blendfiles / 'textures/Bricks/brick_dotted_04-bump.jpg',
extpath / 'textures/Bricks/brick_dotted_04-bump.jpg'),
]
cb.transfer_file.assert_has_calls(expected_calls, any_order=True)
self.assertEqual(len(expected_calls), cb.transfer_file.call_count)
self.assertEqual(0, cb.transfer_file_skipped.call_count)
self.assertGreaterEqual(
cb.transfer_progress.call_count, 6,
'transfer_progress() should be called at least once per asset')
self.assertEqual(0, cb.missing_file.call_count)
def test_rewrite_sequence(self):
ppath = self.blendfiles / 'subdir'
infile = ppath / 'image_sequence_dir_up.blend'
with pack.Packer(infile, ppath, self.tpath) as packer:
packer.strategise()
packer.execute()
bf = blendfile.open_cached(self.tpath / infile.name,
assert_cached=False)
scene = bf.code_index[b'SC'][0]
ed = scene.get_pointer(b'ed')
seq = ed.get_pointer((b'seqbase', b'first'))
seq_strip = seq.get_pointer(b'strip')
imgseq_path = (self.blendfiles / 'imgseq').absolute()
as_bytes = str(imgseq_path.relative_to(imgseq_path.anchor)).encode()
relpath = bpathlib.BlendPath(b'//_outside_project') / as_bytes
# The image sequence base path should be rewritten.
self.assertEqual(b'SQ000210.png', seq[b'name'])
self.assertEqual(relpath, seq_strip[b'dir'])
def pack(self):
print('Packing', self.blend_path, 'into', self.target_path)
try:
bpath = pathlib.Path(self.blend_path)
bpath = bpathlib.make_absolute(bpath)
project_path = bpathlib.make_absolute(bpath).parent
packer = pack.Packer(bpath, project_path, self.target_path)
packer.strategise()
for missing_file in packer.missing_files:
self.__signal_missing_file(str(missing_file))
self.__signal_pack_start()
packer.execute()
self.__signal_pack_end(True)
except Exception as error:
self.__signal_pack_error(str(error))
self.__signal_pack_end(False)
sys.exit(1)
def test_strategise_no_rewrite_required(self):
infile = self.blendfiles / 'doubly_linked.blend'
packer = pack.Packer(infile, self.blendfiles, self.tpath)
packer.strategise()
packed_files = (
'doubly_linked.blend',
'linked_cube.blend',
'basic_file.blend',
'material_textures.blend',
'textures/Bricks/brick_dotted_04-bump.jpg',
'textures/Bricks/brick_dotted_04-color.jpg',
)
for pf in packed_files:
path = self.blendfiles / pf
act = packer._actions[path]
self.assertEqual(pack.PathAction.KEEP_PATH, act.path_action,
'for %s' % pf)
self.assertEqual(self.tpath / pf, act.new_path, 'for %s' % pf)
self.assertEqual({}, self.rewrites(packer))
self.assertEqual(len(packed_files), len(packer._actions))
def test_strategise(self):
cb = mock.Mock(progress.Callback)
infile = self.blendfiles / 'subdir/doubly_linked_up.blend'
with pack.Packer(infile, self.blendfiles, self.tpath) as packer:
packer.progress_cb = cb
packer.strategise()
self.assertEqual(1, cb.pack_start.call_count)
self.assertEqual(0, cb.pack_done.call_count)
expected_calls = [
mock.call(self.blendfiles / 'subdir/doubly_linked_up.blend'),
mock.call(self.blendfiles / 'linked_cube.blend'),
mock.call(self.blendfiles / 'basic_file.blend'),
mock.call(self.blendfiles / 'material_textures.blend'),
]
cb.trace_blendfile.assert_has_calls(expected_calls, any_order=True)
self.assertEqual(len(expected_calls), cb.trace_blendfile.call_count)
expected_calls = [
mock.call(self.blendfiles / 'linked_cube.blend'),
mock.call(self.blendfiles / 'basic_file.blend'),
mock.call(self.blendfiles / 'material_textures.blend'),
mock.call(self.blendfiles /
'textures/Bricks/brick_dotted_04-color.jpg'),
mock.call(self.blendfiles /
'textures/Bricks/brick_dotted_04-bump.jpg'),
]
cb.trace_asset.assert_has_calls(expected_calls, any_order=True)
self.assertEqual(len(expected_calls), cb.trace_asset.call_count)
self.assertEqual(0, cb.rewrite_blendfile.call_count)
self.assertEqual(0, cb.transfer_file.call_count)
self.assertEqual(0, cb.transfer_file_skipped.call_count)
self.assertEqual(0, cb.transfer_progress.call_count)
self.assertEqual(0, cb.missing_file.call_count)
def test_strategise_rewrite(self):
ppath = self.blendfiles / 'subdir'
infile = ppath / 'doubly_linked_up-windows.blend'
packer = pack.Packer(infile, ppath, self.tpath)
packer.strategise()
external_files = (
'linked_cube.blend',
'basic_file.blend',
'material_textures.blend',
'textures/Bricks/brick_dotted_04-bump.jpg',
'textures/Bricks/brick_dotted_04-color.jpg',
)
extpath = self.outside_project()
act = packer._actions[infile]
self.assertEqual(pack.PathAction.KEEP_PATH, act.path_action,
'for %s' % infile.name)
self.assertEqual(self.tpath / infile.name, act.new_path,
'for %s' % infile.name)
for fn in external_files:
path = self.blendfiles / fn
act = packer._actions[path]
self.assertEqual(pack.PathAction.FIND_NEW_LOCATION,
act.path_action, 'for %s' % fn)
self.assertEqual(extpath / fn, act.new_path, 'for %s' % fn)
to_rewrite = (
'linked_cube.blend',
'material_textures.blend',
str(infile.relative_to(self.blendfiles)),
)
rewrites = self.rewrites(packer)
self.assertEqual([self.blendfiles / fn for fn in to_rewrite],
sorted(rewrites.keys()))
# Library link referencing basic_file.blend should (maybe) be rewritten.
rw_linked_cube = rewrites[self.blendfiles / 'linked_cube.blend']
self.assertEqual(1, len(rw_linked_cube))
self.assertEqual(b'LILib', rw_linked_cube[0].block_name)
self.assertEqual(b'//basic_file.blend', rw_linked_cube[0].asset_path)
# Texture links to image assets should (maybe) be rewritten.
rw_mattex = rewrites[self.blendfiles / 'material_textures.blend']
self.assertEqual(2, len(rw_mattex))
rw_mattex.sort() # for repeatable tests
self.assertEqual(b'IMbrick_dotted_04-bump', rw_mattex[0].block_name)
self.assertEqual(b'//textures/Bricks/brick_dotted_04-bump.jpg',
rw_mattex[0].asset_path)
self.assertEqual(b'IMbrick_dotted_04-color', rw_mattex[1].block_name)
self.assertEqual(b'//textures/Bricks/brick_dotted_04-color.jpg',
rw_mattex[1].asset_path)
# Library links from doubly_linked_up.blend to the above to blend files should be rewritten.
rw_dbllink = rewrites[infile]
self.assertEqual(2, len(rw_dbllink))
rw_dbllink.sort() # for repeatable tests
self.assertEqual(b'LILib', rw_dbllink[0].block_name)
self.assertEqual(b'//../linked_cube.blend', rw_dbllink[0].asset_path)
self.assertEqual(b'LILib.002', rw_dbllink[1].block_name)
self.assertEqual(b'//../material_textures.blend',
rw_dbllink[1].asset_path)