def testTaskJson(self):
"""
Test that you can convert a Task to json and back.
"""
class DummyTask(Task):
pass
Task.register("dummy", DummyTask)
dummyTask = Task.create('dummy')
crawlers = FsPath.createFromPath(BaseTestCase.dataDirectory()).glob(['mov'])
targetPaths = []
for crawler in crawlers:
target = '{}_target.mov'.format(crawler.var('name'))
targetPath = os.path.join(BaseTestCase.dataDirectory(), target)
targetPaths.append(targetPath)
dummyTask.add(crawler, targetPath)
jsonResult = dummyTask.toJson()
resultTask = Task.createFromJson(jsonResult)
self.assertCountEqual(dummyTask.optionNames(), resultTask.optionNames())
self.assertCountEqual(dummyTask.metadataNames(), resultTask.metadataNames())
self.assertCountEqual(
map(lambda x: x.var('filePath'), dummyTask.crawlers()),
map(lambda x: x.var('filePath'), resultTask.crawlers())
)
self.assertCountEqual(
map(dummyTask.target, dummyTask.crawlers()),
map(resultTask.target, resultTask.crawlers())
)
def testTaskOutput(self):
"""
Test that task output is returned properly.
"""
class DummyTask(Task):
pass
Task.register("dummy", DummyTask)
dummyTask = Task.create('dummy')
crawlers = FsPath.createFromPath(BaseTestCase.dataDirectory()).glob(['mov'])
targetPaths = []
for crawler in crawlers:
target = '{}_target.mov'.format(crawler.var('name'))
targetPath = os.path.join(BaseTestCase.dataDirectory(), target)
targetPaths.append(targetPath)
crawler.setVar('contextVarTest', 1, True)
dummyTask.add(crawler, targetPath)
result = dummyTask.output()
self.assertEqual(len(result), len(crawlers))
self.assertCountEqual(
map(lambda x: x.var('filePath'), result),
targetPaths
)
self.assertEqual(
list(map(lambda x: x.var('contextVarTest'), result)),
[1]*len(crawlers)
)
for crawler in result:
self.assertIn('contextVarTest', crawler.contextVarNames())
def testUpdateImageMetadata(self):
"""
Test that the UpdateImageMetadata task works properly.
"""
crawler = FsPath.createFromPath(self.__sourcePath)
updateTask = Task.create('updateImageMetadata')
updateTask.add(crawler, self.__targetPath)
result = updateTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
import OpenImageIO as oiio
inputSpec = oiio.ImageInput.open(self.__targetPath).spec()
self.assertEqual(
inputSpec.get_string_attribute("centipede:sourceFile"),
self.__sourcePath)
self.assertEqual(
inputSpec.get_string_attribute("centipede:centipedeUser"),
os.environ['USERNAME'])
checkTask = Task.create('checksum')
checkTask.add(crawler, self.__sourcePath)
self.assertRaises(ChecksumMatchError, checkTask.output)
customMetadata = {"testInt": 0, "testStr": "True"}
UpdateImageMetadata.updateDefaultMetadata(inputSpec, crawler,
customMetadata)
self.assertEqual(inputSpec.get_int_attribute("centipede:testInt"), 0)
self.assertEqual(inputSpec.get_string_attribute("centipede:testStr"),
"True")
def testTaskRegistration(self):
"""
Test that you can register a new Task.
"""
class DummyTask(Task):
pass
Task.register("dummy", DummyTask)
self.assertIn("dummy", Task.registeredNames())
self.assertRaises(TaskTypeNotFoundError, Task.create, 'badTask')
def testExecuteStatus(self):
"""
Test execute status in the task holder.
"""
dummyTask = Task.create('checksum')
crawlers = [FsPath.createFromPath(self.__jsonConfig)]
taskHolder = TaskHolder(dummyTask, Template("{filePath}"))
dummyTask2 = Task.create('checksum')
taskHolder2 = TaskHolder(dummyTask2, Template("{filePath}"))
taskHolder2.setStatus("execute")
taskHolder.addSubTaskHolder(taskHolder2)
self.assertEqual(len(taskHolder.run(crawlers)), len(crawlers) * 2)
def testConvertImage(self):
"""
Test that the ConvertImage task works properly.
"""
crawler = FsPath.createFromPath(self.__sourcePath)
convertTask = Task.create('convertImage')
convertTask.add(crawler, self.__targetPath)
result = convertTask.output()
convertTask = Task.create('convertImage')
convertTask.add(crawler, self.__testPath)
result = convertTask.output()
self.assertEqual(len(result), 1)
checkTask = Task.create('checksum')
checkTask.add(result[0], self.__testPath)
checkTask.output()
def testImageThumbnail(self):
"""
Test that the ImageThumbnail task works properly.
"""
crawler = FsPath.createFromPath(self.__sourcePath)
thumbnailTask = Task.create('imageThumbnail')
thumbnailTask.add(crawler, self.__targetPath)
result = thumbnailTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertEqual(crawler.var("width"), 640)
self.assertEqual(crawler.var("height"), 337)
checkTask = Task.create('checksum')
checkTask.add(crawler, self.__testPath)
checkTask.output()
def testSequenceThumbnail(self):
"""
Test that the SequenceThumbnail task works properly.
"""
thumbnailTask = Task.create('sequenceThumbnail')
sourceFiles = sorted(glob.glob(self.__sourcePath))
for i in map(FsPath.createFromPath, sourceFiles):
thumbnailTask.add(i, self.__targetPath)
result = thumbnailTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertEqual(crawler.var("width"), 640)
self.assertEqual(crawler.var("height"), 360)
checkTask = Task.create('checksum')
checkTask.add(crawler, self.__testPath)
checkTask.output()
def cleanup(self, crawlers):
"""
Remove the data that was copied.
"""
removeTask = Task.create('remove')
for crawler in crawlers:
removeTask.add(crawler, crawler.var("filePath"))
wrapper = TaskWrapper.create('subprocess')
wrapper.setOption('user', '')
wrapper.run(removeTask)
def testResizeImage(self):
"""
Test that the ResizeImage task works properly.
"""
crawler = FsPath.createFromPath(self.__sourcePath)
resizeTask = Task.create('resizeImage')
resizeTask.add(crawler, self.__targetPath)
resizeTask.setOption("width", "480")
resizeTask.setOption("height", "270")
for convertToRGBA in [False, True]:
with self.subTest(convertToRGBA=convertToRGBA):
resizeTask.setOption("convertToRGBA", convertToRGBA)
result = resizeTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertEqual(crawler.var("width"), 480)
self.assertEqual(crawler.var("height"), 270)
checkTask = Task.create('checksum')
checkTask.add(crawler, self.__testPath)
checkTask.output()
def testRemove(self):
"""
Test that the remove task works properly.
"""
crawler = FsPath.createFromPath(self.__path)
removeTask = Task.create('remove')
removeTask.add(crawler, self.__path)
result = removeTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertFalse(os.path.isfile(crawler.var("filePath")))
def testFilterTemplateCrawlers(self):
"""
Test that filter template in task holder.
"""
crawlers = [FsPath.createFromPath(self.__jsonConfig)]
for filterOption in ['0', 'false', 'False']:
with self.subTest(filterOption=filterOption):
dummyTask = Task.create('checksum')
taskHolder = TaskHolder(dummyTask, Template(), Template(filterOption))
result = taskHolder.run(crawlers)
self.assertEqual(len(result), 0)
def testTaskOptions(self):
"""
Test that task options are working properly.
"""
dummyTask = Task.create('copy')
dummyTask.setOption('boolOption', True)
self.assertEqual(dummyTask.option('boolOption'), True)
dummyTask.setOption('floatOption', 1.0)
self.assertEqual(dummyTask.option('floatOption'), 1.0)
dummyTask.setOption('intOption', 1)
self.assertEqual(dummyTask.option('intOption'), 1)
self.assertRaises(TaskInvalidOptionError, dummyTask.option, 'badOption')
def testFilterTemplateNotApplied(self):
"""
Test that filter template should not be applied.
"""
crawlers = [FsPath.createFromPath(self.__jsonConfig)]
for filterOption in ['randomStr', '']:
dummyTask = Task.create('checksum')
with self.subTest(filterOption=filterOption):
taskHolder = TaskHolder(dummyTask, Template("{filePath}"), Template('randomStr'))
result = taskHolder.run(crawlers)
self.assertEqual(len(result), len(crawlers))
def testUpython2(self):
"""
Test that the upython3 subprocess works properly.
"""
resource = Resource.get()
resource.load(self.__taskPath)
crawler = FsPath.createFromPath(self.__sourcePath)
dummyTask = Task.create('uPythonMajorVerTestTask')
dummyTask.add(crawler)
wrapper = TaskWrapper.create("upython2")
result = wrapper.run(dummyTask)
self.assertTrue(len(result), 1)
self.assertEqual(result[0].var("majorVer"), 2)
def _perform(self):
sourceCrawler = self.crawlers()[0]
if self.option('runUPython'):
dummyTask = Task.create('uPythonTestTask')
dummyTask.setOption("runUPython", False)
dummyTask.add(sourceCrawler)
wrapper = TaskWrapper.create('upython')
result = wrapper.run(dummyTask)
else:
import OpenImageIO
sourceCrawler.setVar("testUPython", OpenImageIO.VERSION)
result = [sourceCrawler.clone()]
return result
def testUPython(self):
"""
Test that the UPython subprocess works properly.
"""
resource = Resource.get()
resource.load(self.__taskPath)
crawler = FsPath.createFromPath(self.__sourcePath)
dummyTask = Task.create('uPythonTestTask')
dummyTask.add(crawler)
dummyTask.setOption("runUPython", False)
wrapper = TaskWrapper.create('upython')
result = wrapper.run(dummyTask)
self.assertTrue(len(result), 1)
self.assertIn("testUPython", result[0].varNames())
self.assertEqual(result[0].var("testUPython"), OpenImageIO.VERSION)
def testCopy(self):
"""
Test that the copy task works properly.
"""
crawler = FsPath.createFromPath(self.__sourcePath)
copyTask = Task.create('copy')
copyTask.add(crawler, self.__targetPath)
result = copyTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertEqual(crawler.var("filePath"), self.__targetPath)
self.assertTrue(os.path.isfile(crawler.var("filePath")))
self.assertIsInstance(crawler, Exr)
self.assertEqual(crawler.var("width"), crawler.var("width"))
self.assertEqual(crawler.var("height"), crawler.var("height"))
def testSymlink(self):
"""
Test that hardlinks are skipped when running the chmod task.
"""
link = os.path.join(self.dataDirectory(), 'symlink.exr')
os.symlink(self.__path, link)
self.assertEqual(self.__getPermission(link), '664')
self.assertTrue(os.path.islink(link))
crawler = FsPath.createFromPath(link)
chmodTask = Task.create('chmod')
chmodTask.add(crawler, link)
chmodTask.setOption('directoryMode', '775')
chmodTask.setOption('fileMode', '775')
chmodTask.output()
self.assertEqual(self.__getPermission(link), '664')
self.addCleanup(self.cleanup, link)
def testChmodFile(self):
"""
Test that the chmod task works properly on a file.
"""
crawler = FsPath.createFromPath(self.__path)
chmodTask = Task.create('chmod')
chmodTask.add(crawler, self.__path)
for permission in ["644", "444", "744", "664"]:
with self.subTest(permission=permission):
chmodTask.setOption('directoryMode', permission)
chmodTask.setOption('fileMode', permission)
result = chmodTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertEqual(self.__getPermission(crawler.var('filePath')),
permission)
def testConvertVideo(self):
"""
Test that the Convert Video task works properly.
"""
crawler = FsPath.createFromPath(self.__sourcePath)
convertTask = Task.create('convertVideo')
convertTask.add(crawler, self.__targetPath)
result = convertTask.output()
self.assertEqual(len(result), 1)
# the check is currently done through an approximation
# from the expected size rather than a hash due metadata
# that can vary the file size
convertedSize = os.path.getsize(result[0].var('filePath'))
self.assertGreaterEqual(convertedSize, 1600000)
self.assertLessEqual(convertedSize, 1700000)
def testChmodDir(self):
"""
Test that the chmod task works properly on a directory.
"""
crawler = FsPath.createFromPath(self.__dir)
fileCrawler = FsPath.createFromPath(self.__path)
chmodTask = Task.create('chmod')
chmodTask.add(crawler, self.__dir)
chmodTask.add(fileCrawler, self.__dir)
dirPerm = "775"
filePerm = "664"
chmodTask.setOption('directoryMode', dirPerm)
chmodTask.setOption('fileMode', filePerm)
result = chmodTask.output()
self.assertEqual(len(result), 1)
self.assertEqual(self.__getPermission(self.__dir), dirPerm)
self.assertEqual(self.__getPermission(self.__path), filePerm)
def testTaskClone(self):
"""
Test that cloning tasks works properly.
"""
dummyTask = Task.create('sequenceThumbnail')
crawlers = FsPath.createFromPath(BaseTestCase.dataDirectory()).glob(
['exr'])
for crawler in crawlers:
target = '{}_target'.format(crawler.var('name'))
dummyTask.add(crawler, target)
clone = dummyTask.clone()
self.assertCountEqual(dummyTask.optionNames(), clone.optionNames())
self.assertCountEqual(dummyTask.metadataNames(), clone.metadataNames())
self.assertCountEqual(map(dummyTask.target, dummyTask.crawlers()),
map(clone.target, clone.crawlers()))
self.assertCountEqual(
map(lambda x: x.var('filePath'), dummyTask.crawlers()),
map(lambda x: x.var('filePath'), clone.crawlers()))
def testTaskCrawlers(self):
"""
Test that crawlers are correctly associated with tasks.
"""
dummyTask = Task.create('copy')
crawlers = FsPath.createFromPath(BaseTestCase.dataDirectory()).glob(
['mov'])
for crawler in crawlers:
target = '{}_target'.format(crawler.var('name'))
dummyTask.add(crawler, target)
self.assertEqual(len(dummyTask.crawlers()), len(crawlers))
for filterOption in ['0', 'False', 'false']:
with self.subTest(filterOption=filterOption):
dummyTask.setOption('filterTemplate', filterOption)
self.assertFalse(len(dummyTask.crawlers()))
dummyTask.setOption('filterTemplate', 'randomStr')
self.assertEqual(len(dummyTask.crawlers()), len(crawlers))
for crawler in crawlers:
target = '{}_target'.format(crawler.var('name'))
self.assertEqual(dummyTask.target(crawler), target)
badCrawler = FsPath.createFromPath(self.__jsonConfig)
self.assertRaises(InvalidCrawlerError, dummyTask.target, badCrawler)
from centipede.Task import Task
from centipede.TaskWrapper import TaskWrapper
class UPythonTestTask(Task):
"""
Dummy task for testing UPython subprocess.
"""
def _perform(self):
sourceCrawler = self.crawlers()[0]
if self.option('runUPython'):
dummyTask = Task.create('uPythonTestTask')
dummyTask.setOption("runUPython", False)
dummyTask.add(sourceCrawler)
wrapper = TaskWrapper.create('upython')
result = wrapper.run(dummyTask)
else:
import OpenImageIO
sourceCrawler.setVar("testUPython", OpenImageIO.VERSION)
result = [sourceCrawler.clone()]
return result
Task.register('uPythonTestTask', UPythonTestTask)
import sys
from centipede.Task import Task
class UPythonMajorVerTestTask(Task):
"""
Dummy task for testing python 2 subprocess.
"""
def _perform(self):
sourceCrawler = self.crawlers()[0]
sourceCrawler.setVar("majorVer", sys.version_info[0])
return [sourceCrawler.clone()]
Task.register('uPythonMajorVerTestTask', UPythonMajorVerTestTask)