def testShellScript(self, systemMock):
"""Test a custom shell script layer"""
# The script will be copied into the session directory so we have to create a dummy
# session to use.
layerName = 'arbitrary-layer'
with test_utils.TemporarySessionDirectory(
), tempfile.NamedTemporaryFile() as scriptFile:
scriptContents = '# !/bin/sh\necho zoom zoom zoom'
with open(scriptFile.name, 'w') as fp:
fp.write(scriptContents)
outln = outline.Outline()
outln.setup()
expectedSessionPath = outln.get_session().put_file(scriptFile.name,
layer=layerName,
rename='script')
shellScript = outline.modules.shell.ShellScript(
layerName, script=scriptFile.name)
shellScript.set_outline(outln)
shellScript._setup()
shellScript._execute(FrameSet('5-6'))
with open(expectedSessionPath) as fp:
sessionScriptContents = fp.read()
self.assertEqual(scriptContents, sessionScriptContents)
systemMock.assert_has_calls(
[mock.call(expectedSessionPath, frame=5)])
def submitJob(jobData):
"""Submit the job using the PyOutline API."""
ol = outline.Outline(jobData['name'],
shot=jobData['shot'],
show=jobData['show'],
user=jobData['username'])
lastLayer = None
for layerData in jobData['layers']:
if layerData.layerType == JobTypes.JobTypes.MAYA:
layer = buildMayaLayer(layerData, lastLayer)
elif layerData.layerType == JobTypes.JobTypes.SHELL:
layer = buildShellLayer(layerData, lastLayer)
elif layerData.layerType == JobTypes.JobTypes.NUKE:
layer = buildNukeLayer(layerData, lastLayer)
elif layerData.layerType == JobTypes.JobTypes.BLENDER:
layer = buildBlenderLayer(layerData, lastLayer)
else:
raise ValueError('unrecognized layer type %s' %
layerData.layerType)
ol.add_layer(layer)
lastLayer = layer
if 'facility' in jobData:
ol.set_facility(jobData['facility'])
return outline.cuerun.launch(ol, use_pycuerun=False)
def testFailedShell(self):
"""Test that a failed frame throws an OutlineException in test mode."""
ol = outline.Outline(name="shell_test_v2", current=True)
ol.add_layer(Shell("bah", command=["/bin/lssdsdasdsd"]))
self.assertRaises(outline.OutlineException,
cuerun.launch, ol, range="1", test=True)
def disabled__test_output_passing(self):
"""
Test that output registered in a pre-process is serialized
to a ol:outputs file in the render layer.
"""
with test_utils.TemporarySessionDirectory():
ol = outline.Outline("pre_test")
# the render layer
layer1 = TestA("test1")
# the preprocess
prelayer = outline.layer.LayerPreProcess(layer1)
prelayer._execute = lambda frames: prelayer.add_output(
"test", outline.io.FileSpec("/tmp/foo.#.exr"))
# Add both to the outline
ol.add_layer(layer1)
ol.add_layer(prelayer)
# setup for execute
ol.setup()
# now run the preprocess
prelayer.execute(1000)
# The file should exist.
self.assertTrue(os.path.exists("%s/ol:outputs" %
layer1.get_path()))
# now run a single frame of the render layer and ensure that
# the outputs are automatically loaded.
layer1.execute(1000)
self.assertEquals(1, len(layer1.get_outputs()))
def test_after_init(self):
ol = outline.Outline("after_init")
ol.add_layer(TestAfterInit("test"))
# Ensure after init was run,
self.assertTrue(ol.get_layer("test").get_arg("after_init"))
# Ensure that the layer has the right ol reference
self.assertEquals(ol, ol.get_layer("test").get_outline())
def _makeGpuSpec(self):
ol = outline.Outline(name="spec_version_test")
layer = outline.modules.shell.Shell("test_layer", command=["/bin/ls"])
layer.set_arg("gpus", 4)
layer.set_arg("gpu_memory", 8 * 1024 * 1024)
ol.add_layer(layer)
l = outline.cuerun.OutlineLauncher(ol)
return Et.fromstring(l.serialize())
def test_after_init_current(self):
ol = outline.Outline('after_init', current=True)
TestAfterInit('test')
# Ensure after init was run,
self.assertTrue(ol.get_layer('test').get_arg('after_init'))
# Ensure that the layer has the right ol reference
self.assertEquals(ol, ol.get_layer('test').get_outline())
def _makeMaxCoresGpusSpec(self):
ol = outline.Outline(name="override_max_cores_and_gpus",
maxcores=8,
maxgpus=7)
layer = outline.modules.shell.Shell("test_layer", command=["/bin/ls"])
ol.add_layer(layer)
l = outline.cuerun.OutlineLauncher(ol)
return Et.fromstring(l.serialize())
def testShellScript(self):
fp = open("test.sh", "w")
fp.write("#!/bin/sh\n")
fp.write("echo zoom zoom zoom")
fp.close()
ol = outline.Outline(name="shell_script_test_v1", current=True)
ol.add_layer(ShellScript("script", script="test.sh"))
cuerun.launch(ol, test=True)
def test_dependency_creation(self):
with test_utils.TemporarySessionDirectory():
outline.Outline.current = None
ol = outline.Outline('after_init')
ol.add_layer(TestAfterInit('test'))
ol.add_layer(TestB('testb', require='test'))
ol.setup()
# check the depend was setup properly
self.assertEquals(1, len(ol.get_layer('testb').get_depends()))
def testShortHandDepend(self):
ol = outline.Outline(name="depend_test_v2")
ol.add_layer(Shell("bah1", command=["/bin/ls"]))
ol.add_layer(Shell("bah2", command=["/bin/ls"], require=["bah1:all"]))
job = outline.cuerun.launch(ol, range="1", pause=True)
self.assertEquals(1, job.stats.dependFrames)
cue.test(job)
def setUp(self):
self.ol = outline.Outline()
self.layer = outline.Layer("composite")
self.ol.add_layer(self.layer)
self.layer.add_child(Shell("blah1", command=["ls", "-l"]))
self.layer.add_child(Shell("blah2", command=["ls", "-1"]))
self.layer.add_child(Shell("blah3", command=["ls"]))
self.event = self.ol.get_layer("composite")
def test_dependency_creation(self):
outline.Outline.current = None
ol = outline.Outline("after_init")
ol.add_layer(TestAfterInit("test"))
ol.add_layer(TestB("testb", require="test"))
ol.setup()
# check the depend was setup properly
self.assertEquals(1, len(ol.get_layer("testb").get_depends()))
def test_should_return_default_frame_range(self):
"""Test getting/setting the frame range. If the frame
range is not set on a layer, then it should default to
the outline frame range.
"""
outline.Outline.current = None
ol = outline.Outline('after_init')
layer = outline.layer.Frame('layer-name')
ol.add_layer(layer)
self.assertEqual(outline.layer.DEFAULT_FRAME_RANGE, layer.get_frame_range())
def _makeSpec(self):
# Ensure to reset current
outline.Outline.current = None
ol = outline.Outline(name="spec_version_test")
layer = outline.modules.shell.Shell("test_layer", command=["/bin/ls"])
layer.set_arg("timeout", 420)
layer.set_arg("timeout_llu", 4200)
ol.add_layer(layer)
l = outline.cuerun.OutlineLauncher(ol)
l.set_flag("priority", 42)
return Et.fromstring(l.serialize())
def test_should_create_dependency_from_outline(self):
with test_utils.TemporarySessionDirectory():
outline.Outline.current = None
ol = outline.Outline('after_init')
ol.add_layer(TestAfterInit('test'))
ol.add_layer(TestB('testb', require='test'))
ol.setup()
new_depend = ol.get_layer('testb').get_depends()[0]
self.assertEqual('testb', new_depend.get_dependant_layer().get_name())
self.assertEqual('test', new_depend.get_depend_on_layer().get_name())
def test_add_layer_during_setup(self):
"""Test to ensure that layers added during setup are serialized."""
ol = outline.Outline("mr_hat")
ol.add_layer(TestA("test_a"))
ol.setup()
ol_b = outline.load_outline(ol.get_session().get_file("outline.yaml"))
# ensure the new layer was added
self.assertTrue(ol_b.get_layer("test_b"))
# ensure that setup was actually run on the new layer
self.assertTrue(ol_b.get_layer("test_b").is_setup)
def testShell(self):
"""Test a simple shell command."""
ol = outline.Outline(name="depend_test_v1")
ol.add_layer(Shell("bah1", command=["/bin/ls"]))
ol.add_layer(Shell("bah2", command=["/bin/ls"]))
ol.get_layer("bah1").depend_all("bah2")
job = outline.cuerun.launch(ol, range="1", pause=True)
self.assertEquals(1, job.stats.dependFrames)
cue.test(job)
def test_add_layer_during_setup(self):
"""Test to ensure that layers added during setup are serialized."""
with test_utils.TemporarySessionDirectory():
ol = outline.Outline('mr_hat')
ol.add_layer(TestA('test_a'))
ol.setup()
ol_b = outline.load_outline(ol.get_session().get_file('outline.yaml'))
# ensure the new layer was added
self.assertTrue(ol_b.get_layer('test_b'))
# ensure that setup was actually run on the new layer
self.assertTrue(ol_b.get_layer('test_b').is_setup)
def test_should_return_first_frame_of_outline_range(self):
"""Test getting/setting the frame range. If the frame
range is not set on a layer, then it should default to
the outline frame range.
"""
frame_range = '55-65'
outline.Outline.current = None
ol = outline.Outline('after_init')
ol.set_frame_range(frame_range)
layer = outline.layer.Frame('layer-name')
ol.add_layer(layer)
self.assertEqual('55', layer.get_frame_range())
def test_should_get_dependents(self):
with test_utils.TemporarySessionDirectory():
depend_on_layer = TestAfterInit('test')
depend_er_layer = TestB('testb', require='test')
outline.Outline.current = None
ol = outline.Outline('after_init')
ol.add_layer(depend_on_layer)
ol.add_layer(depend_er_layer)
ol.setup()
depend = depend_on_layer.get_dependents()[0]
self.assertEqual('testb', depend.get_dependant_layer().get_name())
self.assertEqual('test', depend.get_depend_on_layer().get_name())
def testAnyFrameDepend(self):
ol = outline.Outline(name="depend_test_any_frame")
ol.add_layer(Shell("bah1", command=["/bin/ls"], range="1-2"))
ol.add_layer(
Shell("bah2",
command=["/bin/ls"],
require=["bah1:any"],
range="1-1"))
job = outline.cuerun.launch(ol, pause=True)
self.assertEquals(1, job.stats.dependFrames)
cue.test(job)
def setUp(self):
from outline.modules.shell import Shell
self.ol = outline.Outline()
self.layer = outline.Layer("composite")
self.ol.add_layer(self.layer)
self.layer.add_child(Shell("blah1", command=["ls", "-l"]))
self.layer.add_child(Shell("blah2", command=["ls", "-1"]))
self.layer.add_child(Shell("blah3", command=["ls"]))
self.event = self.ol.get_layer("composite")
self.ol.setup()
def testShell(self):
"""Test a simple shell command."""
layer1Name = 'bah1'
layer2Name = 'bah2'
layer1 = Shell(layer1Name, command=['/bin/ls'])
layer2 = Shell(layer2Name, command=['/bin/ls'])
ol = outline.Outline(name='depend_test_v1')
ol.add_layer(layer1)
ol.add_layer(layer2)
ol.get_layer(layer1Name).depend_all(layer2Name)
depends = ol.get_layer(layer1Name).get_depends()
self.assertEqual(1, len(depends))
self.assertEqual(DependType.LayerOnLayer, depends[0].get_type())
self.assertEqual(layer2, depends[0].get_depend_on_layer())
def testShortHandDepend(self):
with test_utils.TemporarySessionDirectory():
layer1Name = 'bah1'
layer2Name = 'bah2'
layer1 = outline.modules.shell.Shell(layer1Name, command=['/bin/ls'])
layer2 = outline.modules.shell.Shell(
layer2Name, command=['/bin/ls'], require=['%s:all' % layer1Name])
ol = outline.Outline(name='depend_test_v2')
ol.add_layer(layer1)
ol.add_layer(layer2)
ol.setup()
depends = ol.get_layer(layer2Name).get_depends()
self.assertEqual(1, len(depends))
self.assertEqual(outline.depend.DependType.LayerOnLayer, depends[0].get_type())
self.assertEqual(layer1, depends[0].get_depend_on_layer())
def testShellSequence(self):
"""Test a simple sequence of shell commands"""
commands = ["/bin/ls"] * 10
ol = outline.Outline(name="shell_sequence_test_v1", current=True)
ol.add_layer(ShellSequence("bah", commands=commands, cores=10, memory="512m"))
job = cuerun.launch(ol, pause=True)
self.assertEquals(10, job.stats.waitingFrames)
self.assertEquals(10, job.stats.pendingFrames)
cue.test(job)
job = opencue.getJob(job)
self.assertEquals(0, job.stats.waitingFrames)
self.assertEquals(10, job.stats.succeededFrames)
def testAnyFrameDepend(self):
with test_utils.TemporarySessionDirectory():
layer1Name = 'bah1'
layer2Name = 'bah2'
layer1 = outline.modules.shell.Shell(layer1Name, command=['/bin/ls'], range='1-2')
layer2 = outline.modules.shell.Shell(layer2Name, command=['/bin/ls'], range='1-1',
require=['%s:any' % layer1Name])
ol = outline.Outline(name='depend_test_any_frame')
ol.add_layer(layer1)
ol.add_layer(layer2)
ol.setup()
depends = ol.get_layer(layer2Name).get_depends()
self.assertEqual(1, len(depends))
self.assertEqual(outline.depend.DependType.FrameByFrame, depends[0].get_type())
self.assertEqual(layer1, depends[0].get_depend_on_layer())
def test_should_call_put_data_on_parent(self):
os.environ = {}
outline.Outline.current = None
ol = outline.Outline('outline-name')
parent_layer_name = 'parent-layer'
parent_layer = outline.layer.Layer(parent_layer_name)
parent_layer.put_data = mock.Mock()
ol.add_layer(parent_layer)
preprocess_layer = outline.layer.LayerPreProcess(parent_layer)
output = outline.io.Path('/path/to/output')
preprocess_layer.add_output('output-name', output)
ol.add_layer(preprocess_layer)
with test_utils.TemporarySessionDirectory():
ol.setup()
preprocess_layer.execute(1)
parent_layer.put_data.assert_called_with(
'ol:outputs', {'output-name': output}, force=True)
def testShellToString(self):
"""Test a string shell command."""
ol = outline.Outline(name="string_test_v1")
ol.add_layer(Shell("string_test", command="/bin/ls -l ./"))
cuerun.launch(ol, range="1", test=True)
def create(self):
ol = outline.Outline()
layer = outline.Layer("test")
ol.add_layer(layer)
return ol, layer
