def collect(obj):
"""
Collect ALL upload files in preparation for submission.
Args:
obj (ConductorJob): The item whose attributes define the scan.
Returns:
PathList: All file dependencies.
"""
result = PathList()
# policy: NONE, GLOB, SMART_SEQUENCE
policy = obj.get_attribute("dependency_scan_policy").get_long()
# If we are not localizing reference contexts, then we need to add the
# referenced project files to the dependencies list.
include_references = not obj.get_attribute("localize_contexts").get_bool()
result.add(*_get_system_dependencies())
result.add(*_get_extra_uploads(obj))
result.add(*get_scan(obj, policy, include_references))
# tell the list to look on disk and expand any "*" or <UDIM> in place
result.glob()
return result
def test_common_path_when_duplicate_entries_of_single_path(self):
d = PathList()
files = [
"/users/joebloggs/tmp/foo.txt", "/users/joebloggs/tmp/foo.txt"
]
d.add(*files)
self.assertEqual(d.common_path(), Path("/users/joebloggs/tmp/foo.txt"))
def _get_output_directory(self):
"""
Get the common path for all image output paths.
NOTE: We don't really need the subpaths any longer because directory
creation is handled in the prerender script. Don't want to mess with
things right now though.
Returns:
dict: common path and list of paths below it
"""
out_paths = PathList()
images = ix.api.OfObjectArray()
self.node.get_attribute("images_and_layers").get_values(images)
for image in images:
directory = os.path.dirname(
image.get_attribute("save_as").get_string())
try:
out_paths.add(directory)
except ValueError as ex:
ix.log_error("{} - while resolving {}".format(
str(ex), directory))
return {
"common_path": out_paths.common_path(),
"output_paths": out_paths
}
def test_next_fails_after_last(self):
d = PathList()
d.add("/file1", "/file2", "/file3")
next(d)
next(d)
next(d)
with self.assertRaises(StopIteration):
next(d)
def test_common_path_when_one_path_is_the_common_path(self):
d = PathList()
files = [
"/users/joebloggs/tmp",
"/users/joebloggs/tmp/bolly/operation",
"/users/joebloggs/tmp/stay/go",
]
d.add(*files)
self.assertEqual(d.common_path(), Path("/users/joebloggs/tmp"))
def test_common_path(self):
d = PathList()
files = [
"/users/joebloggs/tmp/foobar/test",
"/users/joebloggs/tmp/baz/fripp",
"/users/joebloggs/tmp/elephant/corner",
]
d.add(*files)
self.assertEqual(d.common_path(), Path("/users/joebloggs/tmp"))
def test_common_path_when_common_prefix_in_filename(self):
d = PathList()
files = [
"/users/joebloggs/tmp/dissention/perfect",
"/users/joebloggs/tmp/disagreement/crimson",
"/users/joebloggs/tmp/diatribe/belew",
]
d.add(*files)
self.assertEqual(d.common_path(), Path("/users/joebloggs/tmp"))
def test_common_different_drive_letter(self):
d = PathList()
files = [
"D://users/joebloggs/tmp/foo.txt",
"D://users/joebloggs/tmp/modelman.jpg",
"C://users/joebloggs/tmp/ration.cpp",
"C://users/joebloggs/tmp/bill.project",
]
d.add(*files)
self.assertEqual(d.common_path(), Path("/"))
def test_common_path_when_lowest_path_is_the_common_path(self):
d = PathList()
files = [
"/users/joebloggs/tmp/foo.txt",
"/users/joebloggs/tmp/modelman.jpg",
"/users/joebloggs/tmp/ration.cpp",
"/users/joebloggs/tmp/bill.project",
]
d.add(*files)
self.assertEqual(d.common_path(), Path("/users/joebloggs/tmp"))
def _scan_for_references():
result = PathList()
contexts = ix.api.OfContextSet()
ix.application.get_factory().get_root().resolve_all_contexts(contexts)
for context in contexts:
if context.is_reference() and not context.is_disabled():
try:
filename = context.get_attribute("filename").get_string()
result.add(filename)
except ValueError as ex:
ix.log_error(
"{} - while resolving reference {}.filename = {}".format(
str(ex), str(context), filename))
return result
def _get_system_dependencies():
"""
Extracts the destination side of system dependency files.
Returns:
PathList: list of system files to be uploaded
"""
result = PathList()
for entry in system_dependencies():
try:
result.add(entry["dest"])
except ValueError as ex:
ix.log_error("{} - while resolving system_dependency: {}".format(
str(ex), entry["dest"]))
return result
def _validate_images(node):
"""
Check that images or layers are present and set up to be rendered.
Also check that the when there are many output paths, their common path is
not the filesystem root, as this will be the submission's output_path
property.
Args:
node (ConductorJob): Node
"""
images = ix.api.OfObjectArray()
node.get_attribute("images_and_layers").get_values(images)
out_paths = PathList()
if not images.get_count():
ix.log_error(
"No render images. Please reference one or more image items")
for image in images:
if not image.get_attribute("render_to_disk").get_bool():
ix.log_error(
"Image does not have render_to_disk attribute set: {}".format(
image.get_full_name()))
save_path = image.get_attribute("save_as").get_string()
if not save_path:
ix.log_error("Image save_as path is not set: {}".format(
image.get_full_name()))
if save_path.endswith("/"):
ix.log_error("Image save_as path must be a filename, \
not a directory: {}".format(image.get_full_name()))
try:
directory = os.path.dirname(save_path)
out_paths.add(directory)
except ValueError as ex:
ix.log_error("{} - while resolving {}".format(str(ex), directory))
common_path = out_paths.common_path()
paths = "\n".join(p.posix_path() for p in out_paths)
if common_path.depth == 0:
ix.log_error(
"Your output files should be rendered to a common subfolder. Not the filesystem root. {}\n{}"
.format(common_path.posix_path(), paths))
def test_glob_when_files_match_with_questoion_mark(self):
glob.populate(Sequence.create("1-20").expand("/some/file.####.exr"))
d = PathList()
file = "/some/file.00?0.exr"
d.add(file)
d.glob()
self.assertEqual(len(d), 2)
def test_glob_when_files_dont_match(self):
glob.populate(Sequence.create("1-20").expand("/other/file.####.exr"))
d = PathList()
file = "/some/file.*.exr"
d.add(file)
d.glob()
self.assertEqual(len(d), 0)
def test_glob_dedups_when_many_files_match(self):
glob.populate(Sequence.create("1-20").expand("/some/file.####.exr"))
d = PathList()
files = ["/some/file.*.exr", "/some/*.exr"]
d.add(*files)
d.glob()
self.assertEqual(len(d), 20)
def test_next_reset_after_add(self):
d = PathList()
d.add("/file1", "/file2", "/file3")
next(d)
next(d)
d.add("/file4")
self.assertEqual(next(d), Path("/file1"))
def _get_extra_uploads(obj):
"""
Collects any files specified through the extra uploads window.
They are stored in a list attribute on the ConductorJob item.
Args:
obj (ConductorJob): item being processed.
Returns:
PathList: Collected paths.
"""
result = PathList()
extras_attr = obj.get_attribute("extra_uploads")
paths = ix.api.CoreStringArray()
extras_attr.get_values(paths)
for path in paths:
try:
result.add(path)
except ValueError as ex:
ix.log_error("{} - while resolving extra upload path: {}".format(
str(ex), path))
return result
def add_entries(self, entries):
"""
Add a line item for each entry.
Use PathList to deduplicate on the fly. As the addition of
entries may completely change the list (grow or shrink) we
delete and rebuild the list of entries each time a batch is
added.
Args:
entries (list): list of file paths.
"""
if not entries:
return
root_item = self.get_root()
deduped = PathList()
for item in self.item_list:
try:
deduped.add(item.get_name())
except ValueError as ex:
ix.log_error("{} - while resolving {}".format(
str(ex), item.get_name()))
for entry in entries:
try:
deduped.add(entry)
except ValueError as ex:
ix.log_error("{} - while resolving {}".format(str(ex), entry))
# clear existing list
root_item.remove_children()
del self.item_list[:]
for entry in deduped:
item = ix.api.GuiTreeItemBasic(root_item, entry.posix_path())
self.item_list.append(item)
self.refresh()
def test_init_empty(self):
d = PathList()
self.assertEqual(list(d), [])
def get_scan(obj, policy, include_references=True):
"""
Scan all path attrs for dependencies according to the given policy.
If policy is not None: First replace all UDIM tags with a "*" so they may be
globbed.
File sequences may be resolved one of 2 ways:
1. If policy is GLOB, then hashes in filenames will be replaced by a "*" and
globbed later.
2. If policy is SMART then for each filename, we look at its sequence
definition and calculate the frames that will be needed for the frame
range being rendered.
Args:
obj (ConductorJob): Item being processed.
policy (Enum): NONE, GLOB, SMART
include_references (bool, optional): Whether to scan for references. Defaults to True.
Returns:
PathList: Collected paths
"""
result = PathList()
if not policy:
return result
for attr in ix.api.OfAttr.get_path_attrs():
if _should_ignore(attr):
continue
if attr.is_expression_enabled() and attr.is_expression_activated():
filename = _evaluate_static_expression(attr)
else:
filename = attr.get_string()
if not filename:
continue
# always glob for udims.
filename = RX_UDIM.sub("*", filename)
if policy == GLOB:
# replace all frame identifiers with "*" for globbing later
try:
result.add(re.sub((r"(#+|\{frame:\d*d})"), "*", filename))
except ValueError as ex:
ix.log_error("{} - while resolving path: {} = {}".format(
str(ex), str(attr), filename))
else: # SMART
filenames = _smart_expand(obj, attr, filename)
try:
result.add(*filenames)
except ValueError as ex:
ix.log_error("{} - while resolving path: {} = {}".format(
str(ex), str(attr), filename))
# We ignored references in _should_ignore for 2 reasons.
# 1. We only want them if we are not localizing them
# (include_references==True).
# 2. Getting ref contexts from OfAttr.get_path_attrs() is buggy so it's best
# to get them through the root context with resolve_all_contexts()
if include_references:
result.add(*_scan_for_references())
return result
def test_adds_strings(self):
d = PathList()
d.add("/a/file1", "/a/file2")
self.assertEqual(len(d), 2)
def test_dedup_dirtied_on_add(self):
d = PathList()
d.add("/file1")
self.assertFalse(d._clean)
def test_dedup_contained_file(self):
d = PathList()
d.add("/dir1/", "/dir1/file1", "/dir2/file1", "/dir3/file2")
self.assertEqual(len(d), 3)
def test_dedup_same_strings(self):
d = PathList()
d.add("/file1", "/file2", "/file2")
self.assertEqual(len(d), 2)
self.assertIn("/file1", d)
self.assertIn("/file2", d)
def test_dedup_same_paths(self):
d = PathList()
d.add(Path("/file1"), Path("/file2"), Path("/file2"))
self.assertEqual(len(d), 2)
self.assertIn(Path("/file1"), d)
self.assertIn(Path("/file2"), d)
def test_expand_envvar(self):
with mock.patch.dict("os.environ", self.env):
d = PathList()
d.add("$SHOT/file1", "$HOME/file2")
self.assertIn("/metropolis/shot01/file1", d)
self.assertIn("/users/joebloggs/file2", d)
def test_adds_mix(self):
d = PathList()
d.add("/a/file1", "/a/file2", Path("/a/file3"))
self.assertEqual(len(d), 3)
def test_dedup_cleaned_on_access_len(self):
d = PathList()
d.add("/file1")
ls = len(d)
self.assertTrue(d._clean)
def test_expand_tilde(self):
with mock.patch.dict("os.environ", self.env):
d = PathList()
d.add("~/file1", "~/file2")
self.assertIn("/users/joebloggs/file1", d)
def test_adds_paths(self):
d = PathList()
d.add(Path("/a/file1"), Path("/a/file2"))
self.assertEqual(len(d), 2)
