def addBatch(self, new_batch, batch_frames):
""" A batch corresponds to one or more Deadline Tasks
Deadline Tasks must be sequential frames with only a start and end frame
"""
assert (new_batch is None
or type(new_batch) == GafferDispatch.Dispatcher._TaskBatch)
# some TaskNodes like TaskList and TaskWedge submit with no frames because they are just hierarchy placeholders
# they still need to be in for proper dependency handling
if len(batch_frames) > 0:
current_task = GafferDeadline.GafferDeadlineTask(
new_batch,
len(self.getTasks()),
start_frame=batch_frames[0],
end_frame=batch_frames[0])
self._tasks.append(current_task)
for i in range(1, len(batch_frames)):
if (batch_frames[i] - batch_frames[i - 1]) > 1:
current_task = GafferDeadline.GafferDeadlineTask(
new_batch,
len(self.getTasks()),
start_frame=batch_frames[i],
end_frame=batch_frames[i])
self._tasks.append(current_task)
else:
current_task.setEndFrame(batch_frames[i])
else:
# Control nodes like TaskList have no frames but do need tasks created to pass
# through dependencies
self._tasks.append(
GafferDeadline.GafferDeadlineTask(new_batch,
len(self.getTasks())))
def testGafferNode(self):
task_node = GafferDispatch.TaskNode()
dj = GafferDeadline.GafferDeadlineJob()
dj.setGafferNode(task_node)
self.assertEqual(dj.getGafferNode(), task_node)
dj = GafferDeadline.GafferDeadlineJob(gaffer_node=task_node)
self.assertEqual(dj.getGafferNode(), task_node)
self.assertRaises(ValueError, dj.setGafferNode, "bad value")
def testAuxFiles(self):
dj = GafferDeadline.GafferDeadlineJob()
self.assertEqual(dj.getAuxFiles(), [])
dj.setAuxFiles(["file1", "file2"])
self.assertEqual(len(dj.getAuxFiles()), 2)
self.assertIn("file1", dj.getAuxFiles())
dj = GafferDeadline.GafferDeadlineJob(aux_files=["file3", "file4"])
self.assertEqual(len(dj.getAuxFiles()), 2)
self.assertIn("file3", dj.getAuxFiles())
def testParentJob(self):
djc = GafferDeadline.GafferDeadlineJob()
djp = GafferDeadline.GafferDeadlineJob()
task_node = GafferDispatch.TaskNode()
task_node2 = GafferDispatch.TaskNode()
djc.addParentJob(djp)
self.assertIn(djp, djc.getParentJobs())
djp.setGafferNode(task_node)
self.assertEqual(djc.getParentJobByGafferNode(task_node), djp)
self.assertEqual(djc.getParentJobByGafferNode(task_node2), None)
def testPluginProperties(self):
dj = GafferDeadline.GafferDeadlineJob()
self.assertEqual(dj.getPluginProperties(), {})
dj.setPluginProperties({"testProp": "testVal"})
self.assertIn("testProp", dj.getPluginProperties())
self.assertEqual(dj.getPluginProperties()["testProp"], "testVal")
dj = GafferDeadline.GafferDeadlineJob(
plugin_properties={"testProp2": "testVal2"})
self.assertIn("testProp2", dj.getPluginProperties())
self.assertEqual(dj.getPluginProperties()["testProp2"], "testVal2")
def testAuxFiles(self):
dj = GafferDeadline.GafferDeadlineJob(GafferDispatchTest.LoggingTaskNode())
self.assertEqual(dj.getAuxFiles(), [])
dj.setAuxFiles(["file1", "file2"])
self.assertEqual(len(dj.getAuxFiles()), 2)
self.assertIn("file1", dj.getAuxFiles())
dj = GafferDeadline.GafferDeadlineJob(
GafferDispatchTest.LoggingTaskNode(),
auxFiles=["file3", "file4"]
)
self.assertEqual(len(dj.getAuxFiles()), 2)
self.assertIn("file3", dj.getAuxFiles())
def testPluginProperties(self):
dj = GafferDeadline.GafferDeadlineJob(GafferDispatchTest.LoggingTaskNode())
self.assertEqual(dj.getPluginProperties(), {})
dj.setPluginProperties({"testProp": "testVal"})
self.assertIn("testProp", dj.getPluginProperties())
self.assertEqual(dj.getPluginProperties()["testProp"], "testVal")
dj = GafferDeadline.GafferDeadlineJob(
GafferDispatchTest.LoggingTaskNode(),
pluginProperties={"testProp2": "testVal2"}
)
self.assertIn("testProp2", dj.getPluginProperties())
self.assertEqual(dj.getPluginProperties()["testProp2"], "testVal2")
def __dispatcher(self):
dispatcher = GafferDeadline.DeadlineDispatcher()
dispatcher["jobsDirectory"].setValue(
os.path.join(self.temporaryDirectory(), "testJobDirectory").replace("\\", "\\\\")
)
return dispatcher
def _doDispatch(self, root_batch):
'''
_doDispatch is called by Gaffer, the others (prefixed with __) are just helpers for Deadline
Note that Gaffer and Deadline use some terms differently
Gaffer Batch =~ Deadline Task, which could be multiple frames in a single task. Depending on batch layout
multiple Deadline Tasks may be needed to fullfill a single Gaffer Batch. For example, a Deadline
Task can only handle sequential frames.
Gaffer TaskNode =~ Deadline Job. A Gaffer Task can have multiple Deadline Jobs to complete it depending on batch and context layout.
A DeadlineJob is defined by the combination of Gaffer TaskNode and Context.
Gaffer Job = set of Deadline Jobs (could be considered a Deadline Batch)
Use DeadlineJob, DeadlineTask, etc. to denote Deadline terminology and plain Batch, Job, etc. to denote Gaffer terminology.
Batches can have dependencies completely independent of frame numbers. First
walk through the batch tree to build up a set of GafferDeadlineJob objects with GafferDeadlineTask objects corresponding
to the batches.
When all tasks are created, go back through the tree to setup dependencies between tasks. Task dependencies may be different
from Batch dependencies because batches may have been split to accommodate Deadline's sequential frame task requirement.
With dependencies set, start at the leaf nodes of the task tree (no upstream DeadlineJobs) and submit those first. That way
the Deadline Job ID can be stored and used by dependent jobs to set their dependencies correctly.
To be compatible with Deadline's ExtraInfoKeyValue system, dependencies are reformatted at submission as
task:job_dependency_id=task_dependency_number
'''
self._deadline_jobs = []
IECore.Log.info("Beginning Deadline submission")
dispatch_data = {}
dispatch_data["scriptNode"] = root_batch.preTasks()[0].node().scriptNode()
dispatch_data["scriptFile"] = os.path.join(self.jobDirectory(), os.path.basename(dispatch_data["scriptNode"]["fileName"].getValue()) or "untitled.gfr")
dispatch_data["scriptFile"] = dispatch_data["scriptFile"].replace("\\", os.sep).replace("/", os.sep)
dispatch_data["scriptNode"].serialiseToFile(dispatch_data["scriptFile"])
context = Gaffer.Context.current()
dispatch_data["deadlineBatch"] = context.substitute(self["jobName"].getValue()) or "untitled"
root_deadline_job = GafferDeadline.GafferDeadlineJob()
root_deadline_job.setGafferNode(root_batch.node())
root_deadline_job.setAuxFiles([dispatch_data["scriptFile"]])
self.__addGafferDeadlineJob(root_deadline_job)
root_jobs = []
for upstream_batch in root_batch.preTasks():
root_job = self.__buildDeadlineJobWalk(upstream_batch, dispatch_data)
if root_job is not None:
root_jobs.append(root_job)
root_jobs = list(set(root_jobs))
for root_job in root_jobs:
self.__buildDeadlineDependencyWalk(root_job)
# Control jobs with nothing to control should be removed after the dependencies are set up.
# This mostly applies to FrameMask nodes where downstream nodes need to see tasks on the FrameMask
# to trigger Job-Job dependency mode, but those tasks should not be submitted to Deadline.
self.__removeOrphanTasksWalk(root_job)
self.__submitDeadlineJob(root_job, dispatch_data)
def testAddBatch(self):
dj = GafferDeadline.GafferDeadlineJob()
dj.addBatch(None, [1, 2, 3, 4, 5])
dj.addBatch(None, [6, 7, 8, 9, 10])
assert (dj._tasks[0].getStartFrame() == 1)
assert (dj._tasks[0].getEndFrame() == 5)
assert (dj._tasks[1].getStartFrame() == 6)
assert (dj._tasks[1].getEndFrame() == 10)
dj = GafferDeadline.GafferDeadlineJob()
dj.addBatch(None, [1, 2, 3, 7, 8, 9, 100.0, 101.0, 102.0])
assert (len(dj._tasks) == 3)
assert (dj._tasks[0].getTaskNumber() == 0)
assert (dj._tasks[0].getStartFrame() == 1)
assert (dj._tasks[0].getEndFrame() == 3)
assert (dj._tasks[1].getTaskNumber() == 1)
assert (dj._tasks[1].getStartFrame() == 7)
assert (dj._tasks[1].getEndFrame() == 9)
assert (dj._tasks[2].getTaskNumber() == 2)
assert (dj._tasks[2].getStartFrame() == 100)
assert (dj._tasks[2].getEndFrame() == 102)
def testSetFrameRangeFromList(self):
dt = GafferDeadline.GafferDeadlineTask(None, 1)
self.assertRaises(ValueError, dt.setFrameRangeFromList, [0, 4, 9])
self.assertRaises(ValueError, dt.setFrameRangeFromList, [10, 9, 8])
self.assertRaises(ValueError, dt.setFrameRangeFromList, [1.1, 2.0, 3])
self.assertRaises(ValueError, dt.setFrameRangeFromList, [1, 2.2, 3])
dt.setFrameRangeFromList([1, 2, 3, 4])
self.assertEqual(dt.getStartFrame(), 1)
self.assertEqual(dt.getEndFrame(), 4)
dt.setFrameRangeFromList([12.0, 13, 14.0])
self.assertEqual(dt.getStartFrame(), 12)
self.assertEqual(dt.getEndFrame(), 14)
self.assertEqual(type(dt.getStartFrame()), int)
self.assertEqual(type(dt.getEndFrame()), int)
def __buildDeadlineJobWalk(self, batch, dispatch_data):
if batch.blindData().get("deadlineDispatcher:visited"):
return self.__getGafferDeadlineJob(batch.node(), batch.context())
deadline_job = self.__getGafferDeadlineJob(batch.node(), batch.context())
if not deadline_job:
deadline_job = GafferDeadline.GafferDeadlineJob()
deadline_job.setGafferNode(batch.node())
deadline_job.setContext(batch.context())
deadline_job.setAuxFiles([dispatch_data["scriptFile"]])
self.__addGafferDeadlineJob(deadline_job)
deadline_job.addBatch(batch, batch.frames())
for upstream_batch in batch.preTasks():
parent_deadline_job = self.__buildDeadlineJobWalk(upstream_batch, dispatch_data)
if parent_deadline_job is not None:
deadline_job.addParentJob(parent_deadline_job)
batch.blindData()["deadlineDispatcher:visited"] = IECore.BoolData(True)
return deadline_job
def testSetFrameRange(self):
self.assertRaises(ValueError,
GafferDeadline.GafferDeadlineTask,
None,
1,
start_frame=1,
end_frame=0)
dt = GafferDeadline.GafferDeadlineTask(None,
1,
start_frame=0,
end_frame=1)
self.assertRaises(ValueError, dt.setFrameRange, 1, 0)
self.assertRaises(ValueError, dt.setFrameRange, 0.1, 100)
self.assertRaises(ValueError, dt.setFrameRange, 0, 100.1)
dt.setFrameRange(0, 10)
self.assertEqual(dt.getStartFrame(), 0)
self.assertEqual(dt.getEndFrame(), 10)
dt.setFrameRange(55.0, 75.0)
self.assertEqual(dt.getStartFrame(), 55)
self.assertEqual(dt.getEndFrame(), 75)
self.assertEqual(type(dt.getStartFrame()), int)
self.assertEqual(type(dt.getEndFrame()), int)
def testContext(self):
dj = GafferDeadline.GafferDeadlineJob()
self.assertEqual(dj.getContext(), {})
dj.setContext({"test": "value"})
self.assertEqual(dj.getContext(), {"test": "value"})
def testContext(self):
dj = GafferDeadline.GafferDeadlineJob(GafferDispatchTest.LoggingTaskNode())
self.assertEqual(dj.getContext(), Gaffer.Context())
dj.setContext({"test": "value"})
self.assertEqual(dj.getContext(), {"test": "value"})