def _get_render_package_path(self):
"""
Calc the path to the render package.
The name is not always known until
preview/submission time because it is based on the filename and
possibly a timestamp. What this means, practically, is that it won't
show up in the extra uploads window along with
other dependencies when the glob or smart-scan button is pushed.
It will however always show up in the preview window.
We replace spaces in the filename because of a bug in Clarisse
https://www.isotropix.com/user/bugtracker/376
Returns:
string: path
"""
current_filename = ix.application.get_current_project_filename()
path = os.path.splitext(current_filename)[0]
path = os.path.join(os.path.dirname(path),
os.path.basename(path).replace(" ", "_"))
try:
if self.timestamp_render_package:
return Path("{}_ct{}.project".format(path, self.timestamp))
else:
return Path("{}_ct.project".format(path))
except ValueError:
ix.log_error(
'Cannot create a submission from this file: "{}". Has it ever been saved?'
.format(current_filename))
def _get_instance(self):
"""Get everything related to the instance.
Get the machine type, preemptible flag, and number of retries if
preemptible. We use the key from the instance_type menu and look
up the machine spec in the shared data where the full list of
instance_types is stored. When exhaustion API is in effect, the
list of available types may be dynamic, so wetell the user to
refresh.
"""
instance_types = ConductorDataBlock(
product="clarisse").instance_types()
label = self.node.get_attribute(
"instance_type").get_applied_preset_label()
result = {
"preemptible": self.node.get_attribute("preemptible").get_bool(),
"retries": self.node.get_attribute("retries").get_long(),
}
try:
found = next(it for it in instance_types
if str(it['description']) == label)
except StopIteration:
ix.log_error(
"Invalid instance type \"{}\". Try a refresh.".format(label))
result.update(found)
return result
def write_render_package(self):
"""
Write a package suitable for rendering.
A render package is a project file with a special name.
"""
app = ix.application
clarisse_window = app.get_event_window()
self._before_write_package()
current_filename = app.get_current_project_filename()
current_window_title = clarisse_window.get_title()
package_file = self.render_package_path.posix_path()
with cu.disabled_app():
success = ix.application.save_project(package_file)
ix.application.set_current_project_filename(current_filename)
clarisse_window.set_title(current_window_title)
self._after_write_package()
if not success:
ix.log_error(
"Failed to export render package {}".format(package_file))
ix.log_info("Wrote package to {}".format(package_file))
return package_file
def _get_instance(self):
"""Get everything related to the instance.
Get the machine type, preemptible flag, and number of retries if
preemptible. We use the key from the instance_type menu and look
up the machine spec in the shared data where the full list of
instance_types is stored. When exhaustion API is in effect, the
list of available types may be dynamic, so wetell the user to
refresh.
"""
instance_types = ConductorDataBlock(
product="clarisse").instance_types()
label = self.node.get_attribute(
"instance_type").get_applied_preset_label()
result = {
"preemptible": self.node.get_attribute("preemptible").get_bool(),
"retries": self.node.get_attribute("retries").get_long(),
}
try:
found = next(
it for it in instance_types if str(
it['description']) == label)
except StopIteration:
ix.log_error(
"Invalid instance type \"{}\". Try a refresh.".format(label))
result.update(found)
return result
def _get_output_directory(self):
"""
Get the common path for all image output paths.
NOTE: We don't really need the subpaths any longer because directory
creation is handled in the prerender script. Don't want to mess with
things right now though.
Returns:
dict: common path and list of paths below it
"""
out_paths = PathList()
images = ix.api.OfObjectArray()
self.node.get_attribute("images_and_layers").get_values(images)
for image in images:
directory = os.path.dirname(
image.get_attribute("save_as").get_string())
try:
out_paths.add(directory)
except ValueError as ex:
ix.log_error("{} - while resolving {}".format(
str(ex), directory))
return {
"common_path": out_paths.common_path(),
"output_paths": out_paths
}
def _get_sources(self):
"""
Get the images/layers, along with associated Sequence objects.
If we are not rendering a custom range, then the sequence for
each image may be different.
Returns:
list of dict: elements contain an image along with the Sequence that
represents the image range.
"""
images = ix.api.OfObjectArray()
self.node.get_attribute("images_and_layers").get_values(images)
use_custom = self.node.get_attribute("use_custom_frames").get_bool()
# cast to list because OfObjectArray is true even when empty.
if not list(images):
ix.log_error(
"No render images. Please reference one or more image items")
seq = self.sequence["main"]
result = []
for image in images:
if not use_custom:
seq = Sequence.create(*frames_ui.image_range(image))
result.append({"image": image, "sequence": seq})
return result
def _validate_packages(obj):
# for now, just make sure clarisse is present
attr = obj.get_attribute("packages")
paths = ix.api.CoreStringArray()
attr.get_values(paths)
if any(path.startswith('clarisse') for path in paths):
return
ix.log_error("No Clarisse package detected. \
Please use the package chooser to find one.")
def __init__(self, node, parent_tokens, render_package_path):
"""
Build job object for a ConductorJob node.
After _set_tokens has been called, the Job level token variables are
valid and calls to evaluate string attributes will correctly resolve
where those tokens have been used. This is why we evaluate title,
tasks, after the call to _set_tokens()
Args:
node (ConductorJob): item from which to build this job object
parent_tokens (dict): token/value pairs in the scope of the
submission (parent) object.
render_package_path (string): The render project file, which must be
added to this job's dependencies.
"""
self.node = node
self.tasks = []
self.sequence = self._get_sequence()
self.sources = self._get_sources()
self.instance = self._get_instance()
out = self._get_output_directory()
self.common_output_path = out["common_path"]
self.output_paths = out["output_paths"]
tile_width = int(self.node.get_attribute("tiles").get_long())
self.tiles = tile_width * tile_width
self.tokens = self._set_tokens(parent_tokens)
self.clarisse_package = self._get_clarisse_package()
self.environment = self._get_environment()
self.dependencies = deps.collect(self.node)
try:
self.dependencies.add(render_package_path)
except ValueError as ex:
ix.log_error("{} - while resolving {}".format(
str(ex), render_package_path))
expander = Expander(**self.tokens)
self.title = expander.evaluate(
self.node.get_attribute("title").get_string())
# TODO: Add metadata UI
self.metadata = None
task_att = self.node.get_attribute("task_template")
for chunk in self.sequence["main"].chunks():
for tile_number in range(1, self.tiles + 1):
tile_spec = (self.tiles, tile_number)
self.tasks.append(
Task(chunk, task_att, self.sources, tile_spec,
self.tokens))
def _validate_packages(obj):
# for now, just make sure clarisse is present
attr = obj.get_attribute("packages")
paths = ix.api.CoreStringArray()
attr.get_values(paths)
if any(path.startswith('clarisse') for path in paths):
return
ix.log_error(
"No Clarisse package detected. \
Please use the package chooser to find one.")
def _validate_packages(obj):
"""
Make sure a Clarisse version is selected.
Args:
obj (ConductorJob):
"""
if not (obj.get_attribute("clarisse_version").get_applied_preset_label().
startswith("clarisse")):
ix.log_error("No Clarisse package selected.")
def _validate_images(obj):
"""Check some images are present to be rendered.
Then check that they are set up to save to disk.
"""
images = ix.api.OfObjectArray()
obj.get_attribute("images").get_values(images)
if not images.get_count():
ix.log_error(
"No render images. Please reference one or more image items")
for image in images:
if not image.get_attribute("render_to_disk").get_bool():
ix.log_error(
"Image does not have render_to_disk attribute set: {}".format(
image.get_full_name()))
save_path = image.get_attribute("save_as").get_string()
if not save_path:
ix.log_error(
"Image save_as path is not set: {}".format(
image.get_full_name()))
if save_path.endswith("/"):
ix.log_error(
"Image save_as path must be a filename, \
not a directory: {}".format(
image.get_full_name()))
def _scan_for_references():
result = PathList()
contexts = ix.api.OfContextSet()
ix.application.get_factory().get_root().resolve_all_contexts(contexts)
for context in contexts:
if context.is_reference() and not context.is_disabled():
try:
filename = context.get_attribute("filename").get_string()
result.add(filename)
except ValueError as ex:
ix.log_error(
"{} - while resolving reference {}.filename = {}".format(
str(ex), str(context), filename))
return result
def _get_system_dependencies():
"""
Extracts the destination side of system dependency files.
Returns:
PathList: list of system files to be uploaded
"""
result = PathList()
for entry in system_dependencies():
try:
result.add(entry["dest"])
except ValueError as ex:
ix.log_error("{} - while resolving system_dependency: {}".format(
str(ex), entry["dest"]))
return result
def _get_project(self):
"""Get the project from the attr.
Get its ID in case the current project is no longer in the list
of projects at conductor, throw an error.
"""
projects = ConductorDataBlock().projects()
project_att = self.node.get_attribute("conductor_project_name")
label = project_att.get_applied_preset_label()
try:
found = next(p for p in projects if str(p["name"]) == label)
except StopIteration:
ix.log_error(
'Cannot find project "{}" at Conductor.'.format(label))
return {"id": found["id"], "name": str(found["name"])}
def write_render_package(self):
"""Take the value of the render package att and save the file.
A render package is a binary representation of the project
designed for rendering. It must be saved in the same directory
as the project due to the way Clarisse handles relative
dependencies. Currently it does not make references local,
however localized refs are a planned feature for Isotropix, and
for this reason we use this feature rather than send the project
file itself.
"""
success = ix.application.export_render_archive(self.render_package)
if not success:
ix.log_error(
"Failed to export render archive {}".format(
self.render_package))
def _validate_project(obj):
"""
Check the project is set.
Args:
obj (ConductorJob):
"""
projects = ConductorDataBlock().projects()
project_att = obj.get_attribute("conductor_project_name")
label = project_att.get_applied_preset_label()
if label == PROJECT_NOT_SET["name"]:
ix.log_error('Project is not set for "{}".'.format(obj.get_name()))
try:
next(p for p in projects if str(p["name"]) == label)
except StopIteration:
ix.log_error('Cannot find project "{}" at Conductor. \
Please ensure the PROJECT dropdown contains a \
valid project.'.format(label))
def _attribute_sequence(attr, **kw):
"""Get the sequence associated with a filename attribute.
Many attributes have an associated sequence_mode attribute, which
when set to 1 signifies varying frames, and makes availabel start,
end, and offset attributes to help specify the sequence.
If the keyword intersector is given, then work out the intersection
with it. Why? Because during dependency scanning, we can optimize
the number of frames to upload if we use only those frames specified
in the sequence attribute, and intersect them with the frame range
specified in the job node.
"""
intersector = kw.get("intersector")
obj = attr.get_parent_object()
mode_attr = obj.attribute_exists("sequence_mode")
if not (mode_attr and mode_attr.get_long()):
ix.log_error("Attribute is not a sequence mode")
global_frame_rate = ix.application.get_prefs(
ix.api.AppPreferences.MODE_APPLICATION).get_long_value(
"animation", "frames_per_second")
attr_frame_rate = obj.get_attribute("frame_rate").get_long()
if not attr_frame_rate == global_frame_rate:
ix.log_error(
"Can't get attribute sequence when global \
fps is different from fps on the attribute")
start = obj.get_attribute("start_frame").get_long()
end = obj.get_attribute("end_frame").get_long()
if intersector:
# If there's a frame offset on the attribute, then we need to
# do the intersection in the context of that offset.
offset = obj.get_attribute("frame_offset").get_long()
return Sequence.create(start, end, 1).offset(
offset).intersection(intersector).offset(-offset)
return Sequence.create(start, end, 1)
def _get_project(self):
"""Get the project from the attr.
Get its ID in case the current project is no longer in the list
of projects at conductor, throw an error.
"""
projects = ConductorDataBlock(product="clarisse").projects()
project_att = self.node.get_attribute("project")
label = project_att.get_applied_preset_label()
try:
found = next(p for p in projects if str(p["name"]) == label)
except StopIteration:
ix.log_error(
"Cannot find project \"{}\" at Conductor.".format(label))
return {
"id": found["id"],
"name": str(found["name"])
}
def _get_sources(self):
"""Get the images, along with associated Sequence objects.
If we are not rendering a custom range, then the sequence for
each image may be different.
"""
images = ix.api.OfObjectArray()
self.node.get_attribute("images").get_values(images)
use_custom = self.node.get_attribute("use_custom_frames").get_bool()
# cast to list because OfObjectArray is true even when empty.
if not list(images):
ix.log_error(
"No render images. Please reference one or more image items")
seq = self.sequence["main"]
result = []
for image in images:
if not use_custom:
seq = Sequence.create(*frames_ui.image_range(image))
result.append({"image": image, "sequence": seq})
return result
def add_entries(self, entries):
"""
Add a line item for each entry.
Use PathList to deduplicate on the fly. As the addition of
entries may completely change the list (grow or shrink) we
delete and rebuild the list of entries each time a batch is
added.
Args:
entries (list): list of file paths.
"""
if not entries:
return
root_item = self.get_root()
deduped = PathList()
for item in self.item_list:
try:
deduped.add(item.get_name())
except ValueError as ex:
ix.log_error("{} - while resolving {}".format(
str(ex), item.get_name()))
for entry in entries:
try:
deduped.add(entry)
except ValueError as ex:
ix.log_error("{} - while resolving {}".format(str(ex), entry))
# clear existing list
root_item.remove_children()
del self.item_list[:]
for entry in deduped:
item = ix.api.GuiTreeItemBasic(root_item, entry.posix_path())
self.item_list.append(item)
self.refresh()
def _get_extra_uploads(obj):
"""
Collects any files specified through the extra uploads window.
They are stored in a list attribute on the ConductorJob item.
Args:
obj (ConductorJob): item being processed.
Returns:
PathList: Collected paths.
"""
result = PathList()
extras_attr = obj.get_attribute("extra_uploads")
paths = ix.api.CoreStringArray()
extras_attr.get_values(paths)
for path in paths:
try:
result.add(path)
except ValueError as ex:
ix.log_error("{} - while resolving extra upload path: {}".format(
str(ex), path))
return result
def _validate_images(obj):
"""Check some images are present to be rendered.
Then check that they are set up to save to disk.
"""
images = ix.api.OfObjectArray()
obj.get_attribute("images").get_values(images)
if not images.get_count():
ix.log_error(
"No render images. Please reference one or more image items")
for image in images:
if not image.get_attribute("render_to_disk").get_bool():
ix.log_error(
"Image does not have render_to_disk attribute set: {}".format(
image.get_full_name()))
save_path = image.get_attribute("save_as").get_string()
if not save_path:
ix.log_error("Image save_as path is not set: {}".format(
image.get_full_name()))
if save_path.endswith("/"):
ix.log_error("Image save_as path must be a filename, \
not a directory: {}".format(image.get_full_name()))
import ix
app = ix.application
ix.enable_command_history()
working_context = app.get_working_context()
if (not working_context.is_editable()) and working_context.is_content_locked() and working_context.is_remote() :
ix.log_error("Cannot reference project in a locked context.\n")
elif reference_path != "":
clarisse_win = app.get_event_window()
filenames = ix.api.CoreStringVector()
filenames.add(reference_path)
clarisse_win.set_mouse_cursor(ix.api.Gui.MOUSE_CURSOR_WAIT)
app.disable()
ix.reference_file(working_context, filenames)
app.enable()
clarisse_win.set_mouse_cursor(ix.api.Gui.MOUSE_CURSOR_DEFAULT)
ix.disable_command_history()
def get_scan(obj, policy, include_references=True):
"""
Scan all path attrs for dependencies according to the given policy.
If policy is not None: First replace all UDIM tags with a "*" so they may be
globbed.
File sequences may be resolved one of 2 ways:
1. If policy is GLOB, then hashes in filenames will be replaced by a "*" and
globbed later.
2. If policy is SMART then for each filename, we look at its sequence
definition and calculate the frames that will be needed for the frame
range being rendered.
Args:
obj (ConductorJob): Item being processed.
policy (Enum): NONE, GLOB, SMART
include_references (bool, optional): Whether to scan for references. Defaults to True.
Returns:
PathList: Collected paths
"""
result = PathList()
if not policy:
return result
for attr in ix.api.OfAttr.get_path_attrs():
if _should_ignore(attr):
continue
if attr.is_expression_enabled() and attr.is_expression_activated():
filename = _evaluate_static_expression(attr)
else:
filename = attr.get_string()
if not filename:
continue
# always glob for udims.
filename = RX_UDIM.sub("*", filename)
if policy == GLOB:
# replace all frame identifiers with "*" for globbing later
try:
result.add(re.sub((r"(#+|\{frame:\d*d})"), "*", filename))
except ValueError as ex:
ix.log_error("{} - while resolving path: {} = {}".format(
str(ex), str(attr), filename))
else: # SMART
filenames = _smart_expand(obj, attr, filename)
try:
result.add(*filenames)
except ValueError as ex:
ix.log_error("{} - while resolving path: {} = {}".format(
str(ex), str(attr), filename))
# We ignored references in _should_ignore for 2 reasons.
# 1. We only want them if we are not localizing them
# (include_references==True).
# 2. Getting ref contexts from OfAttr.get_path_attrs() is buggy so it's best
# to get them through the root context with resolve_all_contexts()
if include_references:
result.add(*_scan_for_references())
return result
def _validate_images(node):
"""
Check that images or layers are present and set up to be rendered.
Also check that the when there are many output paths, their common path is
not the filesystem root, as this will be the submission's output_path
property.
Args:
node (ConductorJob): Node
"""
images = ix.api.OfObjectArray()
node.get_attribute("images_and_layers").get_values(images)
out_paths = PathList()
if not images.get_count():
ix.log_error(
"No render images. Please reference one or more image items")
for image in images:
if not image.get_attribute("render_to_disk").get_bool():
ix.log_error(
"Image does not have render_to_disk attribute set: {}".format(
image.get_full_name()))
save_path = image.get_attribute("save_as").get_string()
if not save_path:
ix.log_error("Image save_as path is not set: {}".format(
image.get_full_name()))
if save_path.endswith("/"):
ix.log_error("Image save_as path must be a filename, \
not a directory: {}".format(image.get_full_name()))
try:
directory = os.path.dirname(save_path)
out_paths.add(directory)
except ValueError as ex:
ix.log_error("{} - while resolving {}".format(str(ex), directory))
common_path = out_paths.common_path()
paths = "\n".join(p.posix_path() for p in out_paths)
if common_path.depth == 0:
ix.log_error(
"Your output files should be rendered to a common subfolder. Not the filesystem root. {}\n{}"
.format(common_path.posix_path(), paths))
def _attribute_sequence(attr, intersector):
"""
Get the sequence associated with a filename attribute.
Many attributes have an associated sequence_mode attribute, which when set
to 1 signifies varying frames and makes available start, end, and offset
attributes to help specify the sequence.
We work out the intersection of that sequence with the main sequence because
during dependency scanning, we can optimize the number of frames to upload
if we use only those frames specified in the sequence attribute, and
intersect them with the frame range specified in the job node.
Args:
attr (OfAttr): Attribute to query,
intersector (Sequence): The main sequence as defined by the ConductorJob
frames override, or by the images being rendered.
Returns:
dict:
The sequence defined by the attribute intersected with the main sequence.
The sequence the sequence while at the render frames.
"""
obj = attr.get_parent_object()
mode_attr = obj.attribute_exists("sequence_mode")
if not (mode_attr and mode_attr.get_long()):
ix.log_error("Attribute is not in sequence mode {}")
global_frame_rate = ix.application.get_prefs(
ix.api.AppPreferences.MODE_APPLICATION).get_long_value(
"animation", "frames_per_second")
attr_frame_rate = obj.get_attribute("frame_rate").get_long()
if not attr_frame_rate == global_frame_rate:
ix.log_error("Can't get attribute sequence when global \
fps is different from fps on the attribute")
start = obj.get_attribute("start_frame").get_long()
end = obj.get_attribute("end_frame").get_long()
# If there's a frame offset on the attribute, then we need to
# do the intersection in the context of that offset.
# NOTE: Due to a current limitation in Sequence (no negative frame numbers)
# we do a temporary fix using SEQUENCE_OFFSET_KLUDGE. The idea is to do all
# the calcs 100000 frames in the future. When Sequence is fixed and can use
# negative frame numbers, we can revert back to the simpler version which
# can be found in this commit: 542581247d1f109c5511b066f2e7ff5e86577751
offset = obj.get_attribute("frame_offset").get_long()
seq = Sequence.create(start, end,
1).offset(offset + SEQUENCE_OFFSET_KLUDGE)
seq = seq.intersection(intersector.offset(SEQUENCE_OFFSET_KLUDGE))
if not seq:
# The attribute doesn't intersect the render frames
return
# Make a copy of the sequence while at the render frames
render_seq = Sequence.create(str(seq)).offset(-SEQUENCE_OFFSET_KLUDGE)
attr_seq = seq.offset(-(offset + SEQUENCE_OFFSET_KLUDGE))
return {"attr_sequence": attr_seq, "render_sequence": render_seq}