def testConfig(self):
"""
Test that you can run tasks through a config file properly.
"""
taskHolderLoader = JsonLoader()
taskHolderLoader.addFromJsonFile(self.__jsonConfig)
crawlers = FsPath.createFromPath(BaseTestCase.dataDirectory()).glob()
temporaryDir = tempfile.mkdtemp()
dispacher = Dispatcher.create("local")
dispacher.setOption('enableVerboseOutput', False)
dispacher.setOption('awaitExecution', True)
for taskHolder in taskHolderLoader.taskHolders():
taskHolder.addVar('temporaryDir', temporaryDir, True)
dispacher.dispatch(taskHolder, crawlers)
createdCrawlers = FsPath.createFromPath(temporaryDir).glob()
exrCrawlers = list(
filter(lambda x: isinstance(x, Exr), createdCrawlers))
self.assertEqual(len(exrCrawlers), 16)
jpgCrawlers = list(
filter(lambda x: isinstance(x, Jpg), createdCrawlers))
self.assertEqual(len(jpgCrawlers), 1)
self.cleanup(exrCrawlers + jpgCrawlers)
def testTaskJson(self):
"""
Test that you can convert a Task to json and back.
"""
class DummyTask(Task):
pass
Task.register("dummy", DummyTask)
dummyTask = Task.create('dummy')
crawlers = FsPath.createFromPath(BaseTestCase.dataDirectory()).glob(['mov'])
targetPaths = []
for crawler in crawlers:
target = '{}_target.mov'.format(crawler.var('name'))
targetPath = os.path.join(BaseTestCase.dataDirectory(), target)
targetPaths.append(targetPath)
dummyTask.add(crawler, targetPath)
jsonResult = dummyTask.toJson()
resultTask = Task.createFromJson(jsonResult)
self.assertCountEqual(dummyTask.optionNames(), resultTask.optionNames())
self.assertCountEqual(dummyTask.metadataNames(), resultTask.metadataNames())
self.assertCountEqual(
map(lambda x: x.var('filePath'), dummyTask.crawlers()),
map(lambda x: x.var('filePath'), resultTask.crawlers())
)
self.assertCountEqual(
map(dummyTask.target, dummyTask.crawlers()),
map(resultTask.target, resultTask.crawlers())
)
def testXmlVariables(self):
"""
Test that variables are set properly.
"""
crawler = FsPath.createFromPath(self.__xmlFile)
self.assertEqual(crawler.var("type"), "xml")
self.assertEqual(crawler.var("category"), "ascii")
def testConfig(self):
"""
Test that you can run tasks through a config file properly.
"""
taskHolderLoader = JsonLoader()
taskHolderLoader.addFromJsonFile(self.__jsonConfig)
crawlers = FsPath.createFromPath(BaseTestCase.dataDirectory()).glob()
createdCrawlers = []
for taskHolder in taskHolderLoader.taskHolders():
self.assertIn('testCustomVar', taskHolder.varNames())
self.assertEqual(taskHolder.var('testCustomVar'), 'randomValue')
self.assertRaises(TaskHolderInvalidVarNameError, taskHolder.var,
'badVar')
createdCrawlers += taskHolder.run(crawlers)
exrCrawlers = list(
filter(lambda x: isinstance(x, Exr), createdCrawlers))
self.assertEqual(len(exrCrawlers), 16)
jpgCrawlers = list(
filter(lambda x: isinstance(x, Jpg), createdCrawlers))
self.assertEqual(len(jpgCrawlers), 1)
self.cleanup(exrCrawlers + jpgCrawlers)
def testUpdateImageMetadata(self):
"""
Test that the UpdateImageMetadata task works properly.
"""
crawler = FsPath.createFromPath(self.__sourcePath)
updateTask = Task.create('updateImageMetadata')
updateTask.add(crawler, self.__targetPath)
result = updateTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
import OpenImageIO as oiio
inputSpec = oiio.ImageInput.open(self.__targetPath).spec()
self.assertEqual(
inputSpec.get_string_attribute("centipede:sourceFile"),
self.__sourcePath)
self.assertEqual(
inputSpec.get_string_attribute("centipede:centipedeUser"),
os.environ['USERNAME'])
checkTask = Task.create('checksum')
checkTask.add(crawler, self.__sourcePath)
self.assertRaises(ChecksumMatchError, checkTask.output)
customMetadata = {"testInt": 0, "testStr": "True"}
UpdateImageMetadata.updateDefaultMetadata(inputSpec, crawler,
customMetadata)
self.assertEqual(inputSpec.get_int_attribute("centipede:testInt"), 0)
self.assertEqual(inputSpec.get_string_attribute("centipede:testStr"),
"True")
def testTaskOutput(self):
"""
Test that task output is returned properly.
"""
class DummyTask(Task):
pass
Task.register("dummy", DummyTask)
dummyTask = Task.create('dummy')
crawlers = FsPath.createFromPath(BaseTestCase.dataDirectory()).glob(['mov'])
targetPaths = []
for crawler in crawlers:
target = '{}_target.mov'.format(crawler.var('name'))
targetPath = os.path.join(BaseTestCase.dataDirectory(), target)
targetPaths.append(targetPath)
crawler.setVar('contextVarTest', 1, True)
dummyTask.add(crawler, targetPath)
result = dummyTask.output()
self.assertEqual(len(result), len(crawlers))
self.assertCountEqual(
map(lambda x: x.var('filePath'), result),
targetPaths
)
self.assertEqual(
list(map(lambda x: x.var('contextVarTest'), result)),
[1]*len(crawlers)
)
for crawler in result:
self.assertIn('contextVarTest', crawler.contextVarNames())
def testChmodDir(self):
"""
Test that the chmod task works properly on a directory.
"""
crawler = FsPath.createFromPath(self.__dir)
fileCrawler = FsPath.createFromPath(self.__path)
chmodTask = Task.create('chmod')
chmodTask.add(crawler, self.__dir)
chmodTask.add(fileCrawler, self.__dir)
dirPerm = "775"
filePerm = "664"
chmodTask.setOption('directoryMode', dirPerm)
chmodTask.setOption('fileMode', filePerm)
result = chmodTask.output()
self.assertEqual(len(result), 1)
self.assertEqual(self.__getPermission(self.__dir), dirPerm)
self.assertEqual(self.__getPermission(self.__path), filePerm)
def testTemplate(self):
"""
Test that the Template works properly.
"""
crawler = FsPath.createFromPath(self.__file)
value = '(dirname {filePath})/(newver <parent>)/{name}.(pad {frame} 6).{ext}'
result = Template(value).valueFromCrawler(crawler)
self.assertEqual(
result,
os.path.join(BaseTestCase.dataDirectory(), 'v003',
'RND-TST-SHT_lighting_beauty_sr.001001.exr'))
def testRemove(self):
"""
Test that the remove task works properly.
"""
crawler = FsPath.createFromPath(self.__path)
removeTask = Task.create('remove')
removeTask.add(crawler, self.__path)
result = removeTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertFalse(os.path.isfile(crawler.var("filePath")))
def testXmlContents(self):
"""
Test that txt files are parsed properly.
"""
crawler = FsPath.createFromPath(self.__xmlFile)
self.assertEqual(crawler.queryTag('testC')[0], "testing child C")
self.assertEqual(crawler.queryTag('testD1')[0], "1 2 3")
self.assertEqual(
crawler.queryTag('{TestNamespace}testD1',
ignoreNameSpace=False)[0], "1 2 3")
self.assertEqual(crawler.queryTag('testB')[1]['id'], "123")
def testFilterTemplateCrawlers(self):
"""
Test that filter template in task holder.
"""
crawlers = [FsPath.createFromPath(self.__jsonConfig)]
for filterOption in ['0', 'false', 'False']:
with self.subTest(filterOption=filterOption):
dummyTask = Task.create('checksum')
taskHolder = TaskHolder(dummyTask, Template(), Template(filterOption))
result = taskHolder.run(crawlers)
self.assertEqual(len(result), 0)
def testExecuteStatus(self):
"""
Test execute status in the task holder.
"""
dummyTask = Task.create('checksum')
crawlers = [FsPath.createFromPath(self.__jsonConfig)]
taskHolder = TaskHolder(dummyTask, Template("{filePath}"))
dummyTask2 = Task.create('checksum')
taskHolder2 = TaskHolder(dummyTask2, Template("{filePath}"))
taskHolder2.setStatus("execute")
taskHolder.addSubTaskHolder(taskHolder2)
self.assertEqual(len(taskHolder.run(crawlers)), len(crawlers) * 2)
def testTaskCrawlers(self):
"""
Test that crawlers are correctly associated with tasks.
"""
dummyTask = Task.create('copy')
crawlers = FsPath.createFromPath(BaseTestCase.dataDirectory()).glob(
['mov'])
for crawler in crawlers:
target = '{}_target'.format(crawler.var('name'))
dummyTask.add(crawler, target)
self.assertEqual(len(dummyTask.crawlers()), len(crawlers))
for filterOption in ['0', 'False', 'false']:
with self.subTest(filterOption=filterOption):
dummyTask.setOption('filterTemplate', filterOption)
self.assertFalse(len(dummyTask.crawlers()))
dummyTask.setOption('filterTemplate', 'randomStr')
self.assertEqual(len(dummyTask.crawlers()), len(crawlers))
for crawler in crawlers:
target = '{}_target'.format(crawler.var('name'))
self.assertEqual(dummyTask.target(crawler), target)
badCrawler = FsPath.createFromPath(self.__jsonConfig)
self.assertRaises(InvalidCrawlerError, dummyTask.target, badCrawler)
def testFilterTemplateNotApplied(self):
"""
Test that filter template should not be applied.
"""
crawlers = [FsPath.createFromPath(self.__jsonConfig)]
for filterOption in ['randomStr', '']:
dummyTask = Task.create('checksum')
with self.subTest(filterOption=filterOption):
taskHolder = TaskHolder(dummyTask, Template("{filePath}"), Template('randomStr'))
result = taskHolder.run(crawlers)
self.assertEqual(len(result), len(crawlers))
def testUpython2(self):
"""
Test that the upython3 subprocess works properly.
"""
resource = Resource.get()
resource.load(self.__taskPath)
crawler = FsPath.createFromPath(self.__sourcePath)
dummyTask = Task.create('uPythonMajorVerTestTask')
dummyTask.add(crawler)
wrapper = TaskWrapper.create("upython2")
result = wrapper.run(dummyTask)
self.assertTrue(len(result), 1)
self.assertEqual(result[0].var("majorVer"), 2)
def testCopy(self):
"""
Test that the copy task works properly.
"""
crawler = FsPath.createFromPath(self.__sourcePath)
copyTask = Task.create('copy')
copyTask.add(crawler, self.__targetPath)
result = copyTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertEqual(crawler.var("filePath"), self.__targetPath)
self.assertTrue(os.path.isfile(crawler.var("filePath")))
self.assertIsInstance(crawler, Exr)
self.assertEqual(crawler.var("width"), crawler.var("width"))
self.assertEqual(crawler.var("height"), crawler.var("height"))
def testConvertImage(self):
"""
Test that the ConvertImage task works properly.
"""
crawler = FsPath.createFromPath(self.__sourcePath)
convertTask = Task.create('convertImage')
convertTask.add(crawler, self.__targetPath)
result = convertTask.output()
convertTask = Task.create('convertImage')
convertTask.add(crawler, self.__testPath)
result = convertTask.output()
self.assertEqual(len(result), 1)
checkTask = Task.create('checksum')
checkTask.add(result[0], self.__testPath)
checkTask.output()
def testUPython(self):
"""
Test that the UPython subprocess works properly.
"""
resource = Resource.get()
resource.load(self.__taskPath)
crawler = FsPath.createFromPath(self.__sourcePath)
dummyTask = Task.create('uPythonTestTask')
dummyTask.add(crawler)
dummyTask.setOption("runUPython", False)
wrapper = TaskWrapper.create('upython')
result = wrapper.run(dummyTask)
self.assertTrue(len(result), 1)
self.assertIn("testUPython", result[0].varNames())
self.assertEqual(result[0].var("testUPython"), OpenImageIO.VERSION)
def testImageThumbnail(self):
"""
Test that the ImageThumbnail task works properly.
"""
crawler = FsPath.createFromPath(self.__sourcePath)
thumbnailTask = Task.create('imageThumbnail')
thumbnailTask.add(crawler, self.__targetPath)
result = thumbnailTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertEqual(crawler.var("width"), 640)
self.assertEqual(crawler.var("height"), 337)
checkTask = Task.create('checksum')
checkTask.add(crawler, self.__testPath)
checkTask.output()
def testSymlink(self):
"""
Test that hardlinks are skipped when running the chmod task.
"""
link = os.path.join(self.dataDirectory(), 'symlink.exr')
os.symlink(self.__path, link)
self.assertEqual(self.__getPermission(link), '664')
self.assertTrue(os.path.islink(link))
crawler = FsPath.createFromPath(link)
chmodTask = Task.create('chmod')
chmodTask.add(crawler, link)
chmodTask.setOption('directoryMode', '775')
chmodTask.setOption('fileMode', '775')
chmodTask.output()
self.assertEqual(self.__getPermission(link), '664')
self.addCleanup(self.cleanup, link)
def testConvertVideo(self):
"""
Test that the Convert Video task works properly.
"""
crawler = FsPath.createFromPath(self.__sourcePath)
convertTask = Task.create('convertVideo')
convertTask.add(crawler, self.__targetPath)
result = convertTask.output()
self.assertEqual(len(result), 1)
# the check is currently done through an approximation
# from the expected size rather than a hash due metadata
# that can vary the file size
convertedSize = os.path.getsize(result[0].var('filePath'))
self.assertGreaterEqual(convertedSize, 1600000)
self.assertLessEqual(convertedSize, 1700000)
def testChmodFile(self):
"""
Test that the chmod task works properly on a file.
"""
crawler = FsPath.createFromPath(self.__path)
chmodTask = Task.create('chmod')
chmodTask.add(crawler, self.__path)
for permission in ["644", "444", "744", "664"]:
with self.subTest(permission=permission):
chmodTask.setOption('directoryMode', permission)
chmodTask.setOption('fileMode', permission)
result = chmodTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertEqual(self.__getPermission(crawler.var('filePath')),
permission)
def testTaskClone(self):
"""
Test that cloning tasks works properly.
"""
dummyTask = Task.create('sequenceThumbnail')
crawlers = FsPath.createFromPath(BaseTestCase.dataDirectory()).glob(
['exr'])
for crawler in crawlers:
target = '{}_target'.format(crawler.var('name'))
dummyTask.add(crawler, target)
clone = dummyTask.clone()
self.assertCountEqual(dummyTask.optionNames(), clone.optionNames())
self.assertCountEqual(dummyTask.metadataNames(), clone.metadataNames())
self.assertCountEqual(map(dummyTask.target, dummyTask.crawlers()),
map(clone.target, clone.crawlers()))
self.assertCountEqual(
map(lambda x: x.var('filePath'), dummyTask.crawlers()),
map(lambda x: x.var('filePath'), clone.crawlers()))
def testResizeImage(self):
"""
Test that the ResizeImage task works properly.
"""
crawler = FsPath.createFromPath(self.__sourcePath)
resizeTask = Task.create('resizeImage')
resizeTask.add(crawler, self.__targetPath)
resizeTask.setOption("width", "480")
resizeTask.setOption("height", "270")
for convertToRGBA in [False, True]:
with self.subTest(convertToRGBA=convertToRGBA):
resizeTask.setOption("convertToRGBA", convertToRGBA)
result = resizeTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertEqual(crawler.var("width"), 480)
self.assertEqual(crawler.var("height"), 270)
checkTask = Task.create('checksum')
checkTask.add(crawler, self.__testPath)
checkTask.output()
def testXmlCrawler(self):
"""
Test that the Xml crawler test works properly.
"""
crawler = FsPath.createFromPath(self.__xmlFile)
self.assertIsInstance(crawler, Xml)
def testCrawlerCreate(self):
"""
Test that you can create a crawler with a specific type.
"""
crawler = FsPath.createFromPath(self.__turntableFile, "exr")
self.assertIsInstance(crawler, Exr)