def _get_render_package_path(self):
"""
Calc the path to the render package.
The name is not always known until
preview/submission time because it is based on the filename and
possibly a timestamp. What this means, practically, is that it won't
show up in the extra uploads window along with
other dependencies when the glob or smart-scan button is pushed.
It will however always show up in the preview window.
We replace spaces in the filename because of a bug in Clarisse
https://www.isotropix.com/user/bugtracker/376
Returns:
string: path
"""
current_filename = ix.application.get_current_project_filename()
path = os.path.splitext(current_filename)[0]
path = os.path.join(os.path.dirname(path),
os.path.basename(path).replace(" ", "_"))
try:
if self.timestamp_render_package:
return Path("{}_ct{}.project".format(path, self.timestamp))
else:
return Path("{}_ct.project".format(path))
except ValueError:
ix.log_error(
'Cannot create a submission from this file: "{}". Has it ever been saved?'
.format(current_filename))
class PathExpansionTest(unittest.TestCase):
def setUp(self):
self.env = {
"HOME": "/users/joebloggs",
"SHOT": "/metropolis/shot01",
"DEPT": "texturing",
}
def test_posix_tilde_input(self):
with mock.patch.dict("os.environ", self.env):
self.p = Path("~/a/b/c")
self.assertEqual(self.p.posix_path(), "/users/joebloggs/a/b/c")
def test_posix_var_input(self):
with mock.patch.dict("os.environ", self.env):
self.p = Path("$SHOT/a/b/c")
self.assertEqual(self.p.posix_path(), "/metropolis/shot01/a/b/c")
def test_posix_two_var_input(self):
with mock.patch.dict("os.environ", self.env):
self.p = Path("$SHOT/a/b/$DEPT/c")
self.assertEqual(self.p.posix_path(),
"/metropolis/shot01/a/b/texturing/c")
def test_windows_var_input(self):
with mock.patch.dict("os.environ", self.env):
self.p = Path("$HOME\\a\\b\\c")
self.assertEqual(self.p.windows_path(),
"\\users\\joebloggs\\a\\b\\c")
self.assertEqual(self.p.posix_path(), "/users/joebloggs/a/b/c")
class AbsPosixPathTest(unittest.TestCase):
def setUp(self):
self.p = Path("/a/b/c")
def test_posix_path_out(self):
self.assertEqual(self.p.posix_path(), "/a/b/c")
def test_win_path_out(self):
self.assertEqual(self.p.windows_path(), "\\a\\b\\c")
class SpecifyDriveLetterUse(unittest.TestCase):
def test_remove_from_path(self):
self.p = Path("C:\\a\\b\\c")
self.assertEqual(self.p.posix_path(with_drive=False), "/a/b/c")
self.assertEqual(self.p.windows_path(with_drive=False), "\\a\\b\\c")
def test_remove_from_root_path(self):
self.p = Path("C:\\")
self.assertEqual(self.p.posix_path(with_drive=False), "/")
self.assertEqual(self.p.windows_path(with_drive=False), "\\")
class RootPath(unittest.TestCase):
def test_root_path(self):
self.p = Path("/")
self.assertEqual(self.p.posix_path(), "/")
self.assertEqual(self.p.windows_path(), "\\")
def test_drive_letter_root_path(self):
self.p = Path("C:\\")
self.assertEqual(self.p.posix_path(), "C:/")
self.assertEqual(self.p.windows_path(), "C:\\")
class WindowsMixedPathTest(unittest.TestCase):
def test_abs_in_posix_path_out(self):
self.p = Path("\\a\\b\\c/d/e")
self.assertEqual(self.p.posix_path(), "/a/b/c/d/e")
def test_abs_in_windows_path_out(self):
self.p = Path("\\a\\b\\c/d/e")
self.assertEqual(self.p.windows_path(), "\\a\\b\\c\\d\\e")
def test_letter_abs_in_posix_path_out(self):
self.p = Path("C:\\a\\b\\c/d/e")
self.assertEqual(self.p.posix_path(), "C:/a/b/c/d/e")
def test_letter_abs_in_windows_path_out(self):
self.p = Path("C:\\a\\b\\c/d/e")
self.assertEqual(self.p.windows_path(), "C:\\a\\b\\c\\d\\e")
class AbsWindowsPathTest(unittest.TestCase):
def setUp(self):
self.p = Path("C:\\a\\b\\c")
def test_posix_path_out(self):
self.assertEqual(self.p.posix_path(), "C:/a/b/c")
def test_win_path_out(self):
self.assertEqual(self.p.windows_path(), "C:\\a\\b\\c")
# consider just testing on both platforms
def test_os_path_out(self):
with mock.patch("os.name", "posix"):
self.assertEqual(self.p.os_path(), "C:/a/b/c")
with mock.patch("os.name", "nt"):
self.assertEqual(self.p.os_path(), "C:\\a\\b\\c")
def test_common_path_when_duplicate_entries_of_single_path(self):
d = PathList()
files = [
"/users/joebloggs/tmp/foo.txt", "/users/joebloggs/tmp/foo.txt"
]
d.add(*files)
self.assertEqual(d.common_path(), Path("/users/joebloggs/tmp/foo.txt"))
def _set_tokens(self):
"""Env tokens are variables to help the user build expressions.
The user interface has fields for strings such as job title,
task command. The user can use these tokens with <angle brackets> to build those strings. Tokens at the Submission
level are also available in Job level fields, and likewise
tokens at the Job level are available in Task level fields.
"""
tokens = {}
pdir_val = ix.application.get_factory().get_vars().get(
"PDIR").get_string()
tokens["ct_pdir"] = '"{}"'.format(
Path(pdir_val).posix_path(with_drive=False))
tokens["ct_temp_dir"] = "{}".format(
self.tmpdir.posix_path(with_drive=False))
tokens["ct_timestamp"] = self.timestamp
tokens["ct_submitter"] = self.node.get_name()
tokens["ct_render_package"] = '"{}"'.format(
self.render_package_path.posix_path(with_drive=False))
tokens["ct_project"] = self.project["name"]
return tokens
def test_next_reset_after_add(self):
d = PathList()
d.add("/file1", "/file2", "/file3")
next(d)
next(d)
d.add("/file4")
self.assertEqual(next(d), Path("/file1"))
def common_path(self):
"""Find the common path among entries.
This is useful for determining output directory when many
renders are rendering to different places.
In the case where
only single path exists, it is not possible to tell from its
name whether it is a file or directory. We don't want this
method to touch the filesystem, that should be someone else's
problem. A trailing slash would be a hint, but the absence of a
trailing slash does not mean its a regular file. Therefore, in
the case of a single file we return it AS-IS and the caller can
then stat to find out for sure.
If no files exist return None.
If the filesystem root is the common path, return root path, which is
not entirely correct on windows with drive letters.
"""
if not self._entries:
return None
def _all_the_same(rhs):
return all(n == rhs[0] for n in rhs[1:])
levels = zip(*[p.all_components for p in self._entries])
common = [x[0] for x in takewhile(_all_the_same, levels)]
return Path(common or "/")
def system_dependencies():
"""
Provides a list of system files to be sent to the render node.
These will be copied to a directory in preparation for uploading.
This is part of a strategy to satisfy 2 constraints.
1. Dont store special logic on the sidecar.
2. Don't make the render command un-runnable on the local machine.
See docs in ct_cnode and ct_prep for more info.
Returns:
list: Each element is a source/destination pair of paths.
[
{"src": "/some/path.ext", "dest": "/other/path.ext"},
...
]
"""
result = []
conductor_scripts_directory = os.path.join(
os.environ["CONDUCTOR_LOCATION"], "conductor", "clarisse", "scripts")
conductor_tmp_dir = os.path.join(
ix.application.get_factory().get_vars().get("CTEMP").get_string(),
"conductor")
for script in CONDUCTOR_SCRIPTS:
src_path = Path(os.path.join(conductor_scripts_directory,
script)).posix_path()
dest_path = Path(os.path.join(conductor_tmp_dir, script)).posix_path()
result.append({"src": src_path, "dest": dest_path})
config_dir = (ix.application.get_factory().get_vars().get(
"CLARISSE_USER_CONFIG_DIR").get_string())
config_src_file = Path(os.path.join(config_dir,
CLARISSE_CFG_FILENAME)).posix_path()
config_dest_file = Path(
os.path.join(conductor_tmp_dir, CLARISSE_CFG_FILENAME)).posix_path()
result.append({"src": config_src_file, "dest": config_dest_file})
return result
def test_common_path_when_one_path_is_the_common_path(self):
d = PathList()
files = [
"/users/joebloggs/tmp",
"/users/joebloggs/tmp/bolly/operation",
"/users/joebloggs/tmp/stay/go",
]
d.add(*files)
self.assertEqual(d.common_path(), Path("/users/joebloggs/tmp"))
def test_common_path(self):
d = PathList()
files = [
"/users/joebloggs/tmp/foobar/test",
"/users/joebloggs/tmp/baz/fripp",
"/users/joebloggs/tmp/elephant/corner",
]
d.add(*files)
self.assertEqual(d.common_path(), Path("/users/joebloggs/tmp"))
def test_common_path_when_common_prefix_in_filename(self):
d = PathList()
files = [
"/users/joebloggs/tmp/dissention/perfect",
"/users/joebloggs/tmp/disagreement/crimson",
"/users/joebloggs/tmp/diatribe/belew",
]
d.add(*files)
self.assertEqual(d.common_path(), Path("/users/joebloggs/tmp"))
def __init__(self, obj):
"""
Collect data from the Clarisse UI.
Collect attribute values that are common to all jobs, then call
_set_tokens(). After _set_tokens has been called, the Submission level
token variables are valid and calls to evaluate expressions will
correctly resolve where those tokens have been used.
"""
self.node = obj
if self.node.is_kindof("ConductorJob"):
self.nodes = [obj]
else:
raise NotImplementedError
self.localize_before_ship = self.node.get_attribute(
"localize_contexts").get_bool()
self.project_filename = ix.application.get_current_project_filename()
self.timestamp = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
self.timestamp_render_package = self.node.get_attribute(
"timestamp_render_package").get_bool()
self.tmpdir = Path(
os.path.join(
ix.application.get_factory().get_vars().get(
"CTEMP").get_string(),
"conductor",
))
self.render_package_path = self._get_render_package_path()
self.should_delete_render_package = self.node.get_attribute(
"clean_up_render_package").get_bool()
self.local_upload = self.node.get_attribute("local_upload").get_bool()
self.force_upload = self.node.get_attribute("force_upload").get_bool()
self.upload_only = self.node.get_attribute("upload_only").get_bool()
self.project = self._get_project()
self.notifications = self._get_notifications()
self.tokens = self._set_tokens()
self.jobs = []
for node in self.nodes:
job = Job(node, self.tokens, self.render_package_path)
self.jobs.append(job)
def test_common_path_when_lowest_path_is_the_common_path(self):
d = PathList()
files = [
"/users/joebloggs/tmp/foo.txt",
"/users/joebloggs/tmp/modelman.jpg",
"/users/joebloggs/tmp/ration.cpp",
"/users/joebloggs/tmp/bill.project",
]
d.add(*files)
self.assertEqual(d.common_path(), Path("/users/joebloggs/tmp"))
def test_common_different_drive_letter(self):
d = PathList()
files = [
"D://users/joebloggs/tmp/foo.txt",
"D://users/joebloggs/tmp/modelman.jpg",
"C://users/joebloggs/tmp/ration.cpp",
"C://users/joebloggs/tmp/bill.project",
]
d.add(*files)
self.assertEqual(d.common_path(), Path("/"))
class PathContextExpansionTest(unittest.TestCase):
def setUp(self):
self.env = {
"HOME": "/users/joebloggs",
"SHOT": "/metropolis/shot01",
"DEPT": "texturing",
}
self.context = {
"HOME": "/users/janedoe",
"FOO": "fooval",
"BAR_FLY1_": "bar_fly1_val",
"ROOT_DIR": "/some/root",
}
def test_path_replaces_context(self):
self.p = Path("$ROOT_DIR/thefile.jpg", context=self.context)
self.assertEqual(self.p.posix_path(), "/some/root/thefile.jpg")
def test_path_replaces_multiple_context(self):
self.p = Path("$ROOT_DIR/$BAR_FLY1_/thefile.jpg", context=self.context)
self.assertEqual(self.p.posix_path(),
"/some/root/bar_fly1_val/thefile.jpg")
def test_path_context_overrides_env(self):
self.p = Path("$HOME/thefile.jpg", context=self.context)
self.assertEqual(self.p.posix_path(), "/users/janedoe/thefile.jpg")
def test_path_leave_unknown_variable_in_tact(self):
self.p = Path("$ROOT_DIR/$BAR_FLY1_/$FOO/thefile.$F.jpg",
context=self.context)
self.assertEqual(self.p.posix_path(),
"/some/root/bar_fly1_val/fooval/thefile.$F.jpg")
def test_relative_path_var_fails(self):
with self.assertRaises(ValueError):
self.p = Path("$FOO/a/b/c", context=self.context)
def _add_one(self, path):
"""Add a single file.
Note that when an element is added, it may cause the list to
change next time it is deduplicated, which includes getting
shorter. This could happen if a containing directory is added.
Therefore we have to set the peg position to zero.
"""
if not type(path).__name__ == "Path":
path = Path(path)
self._entries.append(path)
self._clean = False
self._current = 0
def glob(self):
"""Glob expansion for entries containing globbable characters.
We don't simply glob every entry since that would remove entries
that don't yet exist. And we can't just rely on zero glob
results because it may have been a legitimate zero result if it
was globbable but matched nothing. So we test for glob
characters (*|?|[) to determine whether to attempt a glob.
"""
self._deduplicate()
result = []
for entry in self._entries:
pp = entry.posix_path()
if GLOBBABLE_REGEX.search(pp):
globs = glob.glob(entry.posix_path())
result += globs
else:
result.append(pp)
self._entries = [Path(g) for g in result]
self._clean = False
self._current = 0
def test_unpacking(self):
d = PathList()
d.add(Path("/a/file1"), Path("/a/file2"))
a, b = d
self.assertEqual(type(a), Path)
def __contains__(self, key):
if not isinstance(key, Path):
key = Path(key)
return key in self._entries
def test_dedup_same_paths(self):
d = PathList()
d.add(Path("/file1"), Path("/file2"), Path("/file2"))
self.assertEqual(len(d), 2)
self.assertIn(Path("/file1"), d)
self.assertIn(Path("/file2"), d)
class Submission(object):
"""
Submission holds all data needed for a submission.
It has potentially many Jobs, and those Jobs each have many Tasks. A
Submission can provide the correct args to send to Conductor, or it can be
used to create a dry run to show the user what will happen.
A Submission also sets a list of tokens that the user can access as <angle
bracket> tokens in order to build strings in the UI such as commands, job
title, and (soon to be added) metadata.
"""
def __init__(self, obj):
"""
Collect data from the Clarisse UI.
Collect attribute values that are common to all jobs, then call
_set_tokens(). After _set_tokens has been called, the Submission level
token variables are valid and calls to evaluate expressions will
correctly resolve where those tokens have been used.
"""
self.node = obj
if self.node.is_kindof("ConductorJob"):
self.nodes = [obj]
else:
raise NotImplementedError
self.localize_before_ship = self.node.get_attribute(
"localize_contexts").get_bool()
self.project_filename = ix.application.get_current_project_filename()
self.timestamp = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
self.timestamp_render_package = self.node.get_attribute(
"timestamp_render_package").get_bool()
self.tmpdir = Path(
os.path.join(
ix.application.get_factory().get_vars().get(
"CTEMP").get_string(),
"conductor",
))
self.render_package_path = self._get_render_package_path()
self.should_delete_render_package = self.node.get_attribute(
"clean_up_render_package").get_bool()
self.local_upload = self.node.get_attribute("local_upload").get_bool()
self.force_upload = self.node.get_attribute("force_upload").get_bool()
self.upload_only = self.node.get_attribute("upload_only").get_bool()
self.project = self._get_project()
self.notifications = self._get_notifications()
self.tokens = self._set_tokens()
self.jobs = []
for node in self.nodes:
job = Job(node, self.tokens, self.render_package_path)
self.jobs.append(job)
def _get_project(self):
"""Get the project from the attr.
Get its ID in case the current project is no longer in the list
of projects at conductor, throw an error.
"""
projects = ConductorDataBlock().projects()
project_att = self.node.get_attribute("conductor_project_name")
label = project_att.get_applied_preset_label()
try:
found = next(p for p in projects if str(p["name"]) == label)
except StopIteration:
ix.log_error(
'Cannot find project "{}" at Conductor.'.format(label))
return {"id": found["id"], "name": str(found["name"])}
def _get_notifications(self):
"""Get notification prefs."""
if not self.node.get_attribute("notify").get_bool():
return None
emails = self.node.get_attribute("email_addresses").get_string()
return [email.strip() for email in emails.split(",") if email.strip()]
def _set_tokens(self):
"""Env tokens are variables to help the user build expressions.
The user interface has fields for strings such as job title,
task command. The user can use these tokens with <angle brackets> to build those strings. Tokens at the Submission
level are also available in Job level fields, and likewise
tokens at the Job level are available in Task level fields.
"""
tokens = {}
pdir_val = ix.application.get_factory().get_vars().get(
"PDIR").get_string()
tokens["ct_pdir"] = '"{}"'.format(
Path(pdir_val).posix_path(with_drive=False))
tokens["ct_temp_dir"] = "{}".format(
self.tmpdir.posix_path(with_drive=False))
tokens["ct_timestamp"] = self.timestamp
tokens["ct_submitter"] = self.node.get_name()
tokens["ct_render_package"] = '"{}"'.format(
self.render_package_path.posix_path(with_drive=False))
tokens["ct_project"] = self.project["name"]
return tokens
def _get_render_package_path(self):
"""
Calc the path to the render package.
The name is not always known until
preview/submission time because it is based on the filename and
possibly a timestamp. What this means, practically, is that it won't
show up in the extra uploads window along with
other dependencies when the glob or smart-scan button is pushed.
It will however always show up in the preview window.
We replace spaces in the filename because of a bug in Clarisse
https://www.isotropix.com/user/bugtracker/376
Returns:
string: path
"""
current_filename = ix.application.get_current_project_filename()
path = os.path.splitext(current_filename)[0]
path = os.path.join(os.path.dirname(path),
os.path.basename(path).replace(" ", "_"))
try:
if self.timestamp_render_package:
return Path("{}_ct{}.project".format(path, self.timestamp))
else:
return Path("{}_ct.project".format(path))
except ValueError:
ix.log_error(
'Cannot create a submission from this file: "{}". Has it ever been saved?'
.format(current_filename))
def get_args(self):
"""
Prepare the args for submission to conductor.
Returns:
list: list of dicts containing submission args per job.
"""
result = []
submission_args = {}
submission_args["local_upload"] = self.local_upload
submission_args["upload_only"] = self.upload_only
submission_args["force"] = self.force_upload
submission_args["project"] = self.project["name"]
submission_args["notify"] = self.notifications
for job in self.jobs:
args = job.get_args(self.upload_only)
args.update(submission_args)
result.append(args)
return result
def submit(self):
"""
Submit all jobs.
Returns:
list: list of response dictionaries, containing response codes
and descriptions.
"""
submission_args = self.get_args()
self.write_render_package()
do_submission, submission_args = self.legalize_upload_paths(
submission_args)
results = []
if do_submission:
for job_args in submission_args:
try:
remote_job = conductor_submit.Submit(job_args)
response, response_code = remote_job.main()
results.append({
"code": response_code,
"response": response
})
except BaseException:
results.append({
"code":
"undefined",
"response":
"".join(traceback.format_exception(*sys.exc_info())),
})
for result in results:
ix.log_info(result)
else:
return [{
"code": "undefined",
"response": "Submission cancelled by user"
}]
self._after_submit()
return results
def write_render_package(self):
"""
Write a package suitable for rendering.
A render package is a project file with a special name.
"""
app = ix.application
clarisse_window = app.get_event_window()
self._before_write_package()
current_filename = app.get_current_project_filename()
current_window_title = clarisse_window.get_title()
package_file = self.render_package_path.posix_path()
with cu.disabled_app():
success = ix.application.save_project(package_file)
ix.application.set_current_project_filename(current_filename)
clarisse_window.set_title(current_window_title)
self._after_write_package()
if not success:
ix.log_error(
"Failed to export render package {}".format(package_file))
ix.log_info("Wrote package to {}".format(package_file))
return package_file
def legalize_upload_paths(self, submission_args):
"""
Alert the user of missing files. If the user doesn't want to continue
with missing files, the result will be False. Otherwise it will be True
and the potentially adjusted args are returned.
Args:
submission_args (list): list of job args.
Returns:
tuple (bool, adjusted args):
"""
missing_files = []
for job_args in submission_args:
existing_files = []
for path in job_args["upload_paths"]:
if os.path.exists(path):
existing_files.append(path)
else:
missing_files.append(path)
job_args["upload_paths"] = existing_files
missing_files = sorted(list(set(missing_files)))
if not missing_files_ui.proceed(missing_files):
return (False, [])
return (True, submission_args)
def _before_write_package(self):
"""
Prepare to write render package.
"""
if self.localize_before_ship:
_localize_contexts()
_remove_conductor()
self._prepare_temp_directory()
self._copy_system_dependencies_to_temp()
def _prepare_temp_directory(self):
"""
Make sure the temp directory has a conductor subdirectory.
"""
tmpdir = self.tmpdir.posix_path()
try:
os.makedirs(tmpdir)
except OSError as ex:
if not (ex.errno == errno.EEXIST and os.path.isdir(tmpdir)):
raise
def _copy_system_dependencies_to_temp(self):
"""
Copy over all system dependencies to a tmp folder.
Wrapper scripts, config files etc. The clarisse.cfg file is special. See
../clarisse_config.py
"""
for entry in deps.system_dependencies():
if os.path.isfile(entry["src"]):
if entry["src"].endswith(".cfg"):
safe_config = ccfg.legalize(entry["src"])
with open(entry["dest"], "w") as dest:
dest.write(safe_config)
ix.log_info("Copy with mods {} to {}".format(
entry["src"], entry["dest"]))
else:
ix.log_info("Copy {} to {}".format(entry["src"],
entry["dest"]))
shutil.copy(entry["src"], entry["dest"])
def _after_submit(self):
"""Clean up, and potentially other post submission actions."""
self._delete_render_package()
def _after_write_package(self):
"""
Runs operations after saving the render package.
If we did something destructive, like localize contexts, then
a backup will have been saved and we now reload it. This strategy
is used because Clarisse's undo is broken when it comes to
undoing context localization.
"""
if self.localize_before_ship:
self._revert_to_saved_scene()
def _delete_render_package(self):
"""
Delete the render package from disk if the user wants to.
"""
if self.should_delete_render_package:
render_package_file = self.render_package_path.posix_path()
if os.path.exists(render_package_file):
os.remove(render_package_file)
def _revert_to_saved_scene(self):
"""
If contexts were localized, we will load the saved scene.
"""
with cu.waiting_cursor():
with cu.disabled_app():
ix.application.load_project(self.project_filename)
@property
def node_name(self):
"""node_name."""
return self.node.get_name()
@property
def filename(self):
"""filename."""
return ix.application.get_current_project_filename()
def has_notifications(self):
"""has_notifications."""
return bool(self.notifications)
@property
def email_addresses(self):
"""email_addresses."""
if not self.has_notifications():
return []
return self.notifications["email"]["addresses"]
def test_common_path_is_slash_when_root(self):
d = PathList()
files = ["/users/joebloggs/tmp/foo.txt", "/dev/joebloggs/tmp/foo.txt"]
d.add(*files)
self.assertEqual(d.common_path(), Path("/"))
def test_abs_in_posix_path_out(self):
self.p = Path("\\a\\b\\c/d/e")
self.assertEqual(self.p.posix_path(), "/a/b/c/d/e")
def test_adds_paths(self):
d = PathList()
d.add(Path("/a/file1"), Path("/a/file2"))
self.assertEqual(len(d), 2)
def test_adds_mix(self):
d = PathList()
d.add("/a/file1", "/a/file2", Path("/a/file3"))
self.assertEqual(len(d), 3)
def test_next(self):
d = PathList()
d.add("/file1", "/file2", "/file3")
self.assertEqual(next(d), Path("/file1"))
self.assertEqual(next(d), Path("/file2"))