def testGlobSkipDuplicated(self):
"""
Test that the glob task with the option skip duplicated enabled (default).
"""
crawler1 = FsCrawler.createFromPath(
os.path.join(BaseTestCase.dataTestsDirectory(), "glob",
"test.json"))
crawler2 = FsCrawler.createFromPath(
os.path.join(BaseTestCase.dataTestsDirectory(), "glob",
"text.txt"))
globTask = Task.create('glob')
globTask.add(
crawler1,
Template("(dirname {filePath})/**/*.exr").valueFromCrawler(
crawler1))
globTask.add(
crawler2,
Template("(dirname {filePath})/images/*.exr").valueFromCrawler(
crawler2))
result = globTask.output()
self.assertEqual(len(result), len(self.__globFiles['exr']))
for resultCrawler in result:
self.assertIsInstance(resultCrawler, ExrCrawler)
self.assertIn(resultCrawler.var('baseName'),
self.__globFiles['exr'])
def testFsCrawler(self):
"""
Test that the fs crawler test works.
"""
pathHolder = PathHolder(self.__dir)
self.assertTrue(FsCrawler.test(pathHolder, None))
notAPathHolder = {}
self.assertFalse(FsCrawler.test(notAPathHolder, None))
def testConfig(self):
"""
Test that you can run tasks through a config file properly.
"""
taskHolderLoader = Loader()
taskHolderLoader.loadFromFile(self.__jsonConfig)
crawlers = FsCrawler.createFromPath(
BaseTestCase.dataTestsDirectory()).glob()
createdCrawlers = []
for taskHolder in taskHolderLoader.taskHolders():
self.assertIn('testCustomVar', taskHolder.varNames())
self.assertEqual(taskHolder.var('testCustomVar'), 'randomValue')
self.assertRaises(TaskHolderInvalidVarNameError, taskHolder.var,
'badVar')
createdCrawlers += taskHolder.run(crawlers)
exrCrawlers = list(
filter(lambda x: isinstance(x, ExrCrawler), createdCrawlers))
self.assertEqual(len(exrCrawlers), 16)
jpgCrawlers = list(
filter(lambda x: isinstance(x, JpgCrawler), createdCrawlers))
self.assertEqual(len(jpgCrawlers), 1)
self.cleanup(exrCrawlers + jpgCrawlers)
def testTaskJson(self):
"""
Test that you can convert a Task to json and back.
"""
class DummyTask(Task):
pass
Task.register("dummy", DummyTask)
dummyTask = Task.create('dummy')
crawlers = FsCrawler.createFromPath(
BaseTestCase.dataTestsDirectory()).glob(['mov'])
targetPaths = []
for crawler in crawlers:
target = '{}_target.mov'.format(crawler.var('name'))
targetPath = os.path.join(BaseTestCase.dataTestsDirectory(),
target)
targetPaths.append(targetPath)
dummyTask.add(crawler, targetPath)
jsonResult = dummyTask.toJson()
resultTask = Task.createFromJson(jsonResult)
self.assertCountEqual(dummyTask.optionNames(),
resultTask.optionNames())
self.assertCountEqual(dummyTask.metadataNames(),
resultTask.metadataNames())
self.assertCountEqual(
map(lambda x: x.var('filePath'), dummyTask.crawlers()),
map(lambda x: x.var('filePath'), resultTask.crawlers()))
self.assertCountEqual(map(dummyTask.target, dummyTask.crawlers()),
map(resultTask.target, resultTask.crawlers()))
def testTaskOutput(self):
"""
Test that task output is returned properly.
"""
class DummyTask(Task):
pass
Task.register("dummy", DummyTask)
dummyTask = Task.create('dummy')
crawlers = FsCrawler.createFromPath(
BaseTestCase.dataTestsDirectory()).glob(['mov'])
targetPaths = []
for crawler in crawlers:
target = '{}_target.mov'.format(crawler.var('name'))
targetPath = os.path.join(BaseTestCase.dataTestsDirectory(),
target)
targetPaths.append(targetPath)
crawler.setVar('contextVarTest', 1, True)
dummyTask.add(crawler, targetPath)
result = dummyTask.output()
self.assertEqual(len(result), len(crawlers))
self.assertCountEqual(map(lambda x: x.var('filePath'), result),
targetPaths)
self.assertEqual(list(map(lambda x: x.var('contextVarTest'), result)),
[1] * len(crawlers))
for crawler in result:
self.assertIn('contextVarTest', crawler.contextVarNames())
def testGlobAll(self):
"""
Test that the glob task find all files & directories.
"""
crawler = FsCrawler.createFromPath(
os.path.join(BaseTestCase.dataTestsDirectory(), "glob",
"test.json"))
globTask = Task.create('glob')
globTask.add(
crawler,
Template("(dirname {filePath})/**/*").valueFromCrawler(crawler))
result = globTask.output()
self.assertEqual(
len(result),
len(self.__globFiles['exr']) + len(self.__globFiles['json']) +
len(self.__globFiles['txt']) + len(self.__globFiles['directory']))
for resultCrawler in result:
baseType = 'directory' if 'ext' not in resultCrawler.varNames(
) else resultCrawler.var('ext')
if baseType == 'exr':
CrawlerType = ExrCrawler
elif baseType == 'json':
CrawlerType = JsonCrawler
elif baseType == 'txt':
CrawlerType = TxtCrawler
else:
baseType = 'directory'
CrawlerType = DirectoryCrawler
self.assertIsInstance(resultCrawler, CrawlerType)
self.assertIn(resultCrawler.var('baseName'),
self.__globFiles[baseType])
def testReport(self):
"""
Test the output produced by the reporter.
"""
resource = Resource.get()
resource.load(self.__taskPath)
taskHolderLoader = JsonLoader()
taskHolderLoader.loadFromFile(self.__jsonConfig)
crawlers = FsCrawler.createFromPath(
BaseTestCase.dataTestsDirectory()).glob()
dispacher = Dispatcher.create('local')
dispacher.setOption('defaultReporter', 'columns')
outputStream = io.StringIO()
dispacher.setStdout(outputStream)
dispacher.setOption('enableVerboseOutput', False)
dispacher.setOption('awaitExecution', True)
for taskHolder in taskHolderLoader.taskHolders():
dispacher.dispatch(taskHolder, crawlers)
output = outputStream.getvalue().replace('\r\n', '\n').split('\n')
prefixSize = None
for index, line in enumerate(self.__output.split('\n')[1:-1]):
if prefixSize is None:
prefixSize = len(line) - len(line.lstrip())
line = line[prefixSize:]
outputLine = output[index].replace('\\', '/')
if not fnmatch(outputLine, line):
self.assertEqual(outputLine, line)
def testXmlVariables(self):
"""
Test that variables are set properly.
"""
crawler = FsCrawler.createFromPath(self.__xmlFile)
self.assertEqual(crawler.var("type"), "xml")
self.assertEqual(crawler.var("category"), "ascii")
def testSymlink(self):
"""
Test that symlink support in link task works properly.
"""
crawler = FsCrawler.createFromPath(self.__targetPath)
linkTask = Task.create('link')
linkTask.setOption('type', 'symlink')
linkTask.add(crawler, self.__targetSymlinkPath)
result = None
try:
result = linkTask.output()
except OSError as err:
if str(err).startswith('[WinError 1314]'):
sys.stderr.write('Current user has no privilege for creating symlinks')
if result is None:
return
self.assertEqual(len(result), 1)
self.assertTrue(os.path.exists(result[0].var('filePath')))
if not platform.system() == "Windows":
self.assertTrue(os.path.islink(result[0].var('filePath')))
def testChmodDir(self):
"""
Test that the chmod task works properly on a directory.
"""
crawler = FsCrawler.createFromPath(self.__dir)
fileCrawler = FsCrawler.createFromPath(self.__path)
chmodTask = Task.create('chmod')
chmodTask.add(crawler, self.__dir)
chmodTask.add(fileCrawler, self.__dir)
dirPerm = "775"
filePerm = "664"
chmodTask.setOption('directoryMode', dirPerm)
chmodTask.setOption('fileMode', filePerm)
result = chmodTask.output()
self.assertEqual(len(result), 1)
self.assertEqual(self.__getPermission(self.__dir), dirPerm)
self.assertEqual(self.__getPermission(self.__path), filePerm)
def testConvertImage(self):
"""
Test that the ConvertImage task works properly.
"""
crawler = FsCrawler.createFromPath(self.__sourcePath)
convertTask = Task.create('convertImage')
convertTask.add(crawler, self.__targetPath)
result = convertTask.output()
self.assertEqual(len(result), 1)
self.assertTrue(os.path.exists(result[0].var('filePath')))
def testConfig(self):
"""
Test that you can run tasks through a config file properly.
"""
taskHolderLoader = JsonLoader()
taskHolderLoader.loadFromFile(self.__jsonConfig)
crawlers = FsCrawler.createFromPath(
BaseTestCase.dataTestsDirectory()).glob()
temporaryDir = tempfile.mkdtemp()
dispacher = Dispatcher.create("local")
outputStream = io.StringIO()
dispacher.setStdout(outputStream)
dispacher.setOption('enableVerboseOutput', False)
dispacher.setOption('awaitExecution', True)
for taskHolder in taskHolderLoader.taskHolders():
taskHolder.addVar('temporaryDir', temporaryDir, True)
dispacher.dispatch(taskHolder, crawlers)
createdCrawlers = FsCrawler.createFromPath(temporaryDir).glob()
exrCrawlers = list(
filter(lambda x: isinstance(x, ExrCrawler), createdCrawlers))
self.assertEqual(len(exrCrawlers), 16)
jpgCrawlers = list(
filter(lambda x: isinstance(x, JpgCrawler), createdCrawlers))
self.assertEqual(len(jpgCrawlers), 1)
output = outputStream.getvalue().replace('\r\n', '\n').split('\n')
prefixSize = None
for index, line in enumerate(self.__output.split('\n')[1:-1]):
if prefixSize is None:
prefixSize = len(line) - len(line.lstrip())
line = line[prefixSize:]
outputLine = output[index].replace('\\', '/')
if not fnmatch(outputLine, line):
self.assertEqual(outputLine, line)
self.cleanup(exrCrawlers + jpgCrawlers)
def testRemove(self):
"""
Test that the remove task works properly.
"""
crawler = FsCrawler.createFromPath(self.__path)
removeTask = Task.create('remove')
removeTask.add(crawler, self.__path)
result = removeTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertFalse(os.path.isfile(crawler.var("filePath")))
def testTemplate(self):
"""
Test that the Template works properly.
"""
crawler = FsCrawler.createFromPath(self.__file)
value = '(dirname {filePath})/(newver <parent>)/{name}.(pad {frame} 6).{ext}'
result = Template(value).valueFromCrawler(crawler)
self.assertEqual(
os.path.normpath(result),
os.path.join(BaseTestCase.dataTestsDirectory(), 'v003',
'RND-TST-SHT_lighting_beauty_sr.001001.exr'))
def testNestProcedureTemplateSimple(self):
"""
Test simple nested procedures in the template.
"""
crawler = FsCrawler.createFromPath(self.__file)
value = "/a/b/c/(dirname(dirname '/d/e/f'))/(newver <parent>)/{name}.(pad {frame} 6).{ext}"
result = Template(value).valueFromCrawler(crawler)
self.assertEqual(
os.path.normpath(result),
os.path.normpath(
'/a/b/c/d/v001/RND-TST-SHT_lighting_beauty_sr.001001.exr'))
def testCcVariables(self):
"""
Test that variables are set properly.
"""
crawler = FsCrawler.create(PathHolder(self.__ccFile))
self.assertEqual(crawler.var("type"), "cc")
self.assertEqual(crawler.var("category"), "lut")
self.assertEqual(crawler.var("slope"), [1.2, 1.3, 1.4])
self.assertEqual(crawler.var("offset"), [0.2, 0.3, 0.4])
self.assertEqual(crawler.var("power"), [1.5, 1.6, 1.7])
self.assertEqual(crawler.var("saturation"), 1.0)
def testXmlContents(self):
"""
Test that txt files are parsed properly.
"""
crawler = FsCrawler.createFromPath(self.__xmlFile)
self.assertEqual(crawler.queryTag('testC')[0], "testing child C")
self.assertEqual(crawler.queryTag('testD1')[0], "1 2 3")
self.assertEqual(
crawler.queryTag('{TestNamespace}testD1',
ignoreNameSpace=False)[0], "1 2 3")
self.assertEqual(crawler.queryTag('testB')[1]['id'], "123")
def testCdlVariables(self):
"""
Test that variables are set properly.
"""
crawler = FsCrawler.create(PathHolder(self.__cdlFile))
self.assertEqual(crawler.var("type"), "cdl")
self.assertEqual(crawler.var("category"), "lut")
self.assertEqual(crawler.var("slope"), [1.014, 1.0104, 0.62])
self.assertEqual(crawler.var("offset"), [-0.00315, -0.00124, 0.3103])
self.assertEqual(crawler.var("power"), [1.0, 0.9983, 1.0])
self.assertEqual(crawler.var("saturation"), 1.09)
def testImageThumbnail(self):
"""
Test that the ImageThumbnail task works properly.
"""
crawler = FsCrawler.createFromPath(self.__sourcePath)
thumbnailTask = Task.create('imageThumbnail')
thumbnailTask.add(crawler, self.__targetPath)
result = thumbnailTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertEqual(crawler.var("width"), 640)
self.assertEqual(crawler.var("height"), 360)
def testChecksumTask(self):
"""
Test that the checksum task works properly.
"""
crawler = FsCrawler.createFromPath(self.__sourcePath)
checksumTask = Task.create('checksum')
checksumTask.add(crawler, self.__targetPath)
result = checksumTask.output()
self.assertEqual(len(result), 1)
checksumTask = Task.create('checksum')
checksumTask.add(crawler, self.__otherPath)
self.assertRaises(ChecksumTaskMatchError, checksumTask.output)
def testNestProcedureTemplateMultiple(self):
"""
Test multiple nested procedures in the template.
"""
crawler = FsCrawler.createFromPath(self.__file)
value = "/a/b/c/(concat (dirname(dirname (dirname '/d/e/f/g'))) '_' (dirname (dirname {var})))/(newver <parent>)/{name}.(pad {frame} 6).{ext}"
result = Template(value).valueFromCrawler(crawler,
extraVars={'var': 'h/j/l'})
self.assertEqual(
os.path.normpath(result),
os.path.normpath(
'/a/b/c/d_h/v001/RND-TST-SHT_lighting_beauty_sr.001001.exr'))
def testFilterTemplateCrawlers(self):
"""
Test that filter template in task holder.
"""
crawlers = [FsCrawler.createFromPath(self.__jsonConfig)]
for filterOption in ['0', 'false', 'False']:
dummyTask = Task.create('checksum')
taskHolder = TaskHolder(dummyTask, Template(),
Template(filterOption))
result = taskHolder.run(crawlers)
self.assertEqual(len(result), 0)
def testConvertTexture(self):
"""
Test that the ConvertTexture task works properly.
"""
crawler = FsCrawler.createFromPath(self.__sourcePath)
convertTask = Task.create('convertTexture')
convertTask.setOption('maketxArgs', '-u --unpremult --oiio')
convertTask.add(crawler, self.__targetPath)
result = convertTask.output()
self.assertEqual(len(result), 1)
self.assertEqual(result[0].var('filePath'), self.__targetPath)
self.assertTrue(os.path.exists(self.__targetPath))
def testHardlink(self):
"""
Test that hardlink support in link task works properly.
"""
crawler = FsCrawler.createFromPath(self.__targetPath)
linkTask = Task.create('link')
linkTask.setOption('type', 'hardlink')
linkTask.add(crawler, self.__targetHardlinkPath)
result = linkTask.output()
self.assertEqual(len(result), 1)
self.assertTrue(os.path.exists(result[0].var('filePath')))
self.assertFalse(os.path.islink(result[0].var('filePath')))
def testFilterTemplateNotApplied(self):
"""
Test that filter template should not be applied.
"""
crawlers = [FsCrawler.createFromPath(self.__jsonConfig)]
for filterOption in ['randomStr', '']:
dummyTask = Task.create('checksum')
taskHolder = TaskHolder(dummyTask, Template("{filePath}"),
Template('randomStr'))
result = taskHolder.run(crawlers)
self.assertEqual(len(result), len(crawlers))
def testExecuteStatus(self):
"""
Test execute status in the task holder.
"""
dummyTask = Task.create('checksum')
crawlers = [FsCrawler.createFromPath(self.__jsonConfig)]
taskHolder = TaskHolder(dummyTask, Template("{filePath}"))
dummyTask2 = Task.create('checksum')
taskHolder2 = TaskHolder(dummyTask2, Template("{filePath}"))
taskHolder2.setStatus("execute")
taskHolder.addSubTaskHolder(taskHolder2)
self.assertEqual(len(taskHolder.run(crawlers)), len(crawlers) * 2)
def testPython2(self):
"""
Test that the python3 subprocess works properly.
"""
resource = Resource.get()
resource.load(self.__taskPath)
crawler = FsCrawler.createFromPath(self.__sourcePath)
dummyTask = Task.create('pythonMajorVerTestTask')
dummyTask.add(crawler)
wrapper = TaskWrapper.create("python2")
result = wrapper.run(dummyTask)
self.assertTrue(len(result), 1)
self.assertEqual(result[0].var("majorVer"), 2)
def testChmodFile(self):
"""
Test that the chmod task works properly on a file.
"""
crawler = FsCrawler.createFromPath(self.__path)
chmodTask = Task.create('chmod')
chmodTask.add(crawler, self.__path)
for permission in ["644", "444", "744", "664"]:
chmodTask.setOption('directoryMode', permission)
chmodTask.setOption('fileMode', permission)
result = chmodTask.output()
self.assertEqual(len(result), 1)
crawler = result[0]
self.assertEqual(self.__getPermission(crawler.var('filePath')),
permission)
def testNestProcedureTemplateArithmetic(self):
"""
Test arithmetic nested procedures in the template.
"""
crawler = FsCrawler.createFromPath(self.__file)
value = "/a/b/c/({a} + (sum {b} 2))/(newver <parent>)/{name}.(pad {frame} 6).{ext}"
result = Template(value).valueFromCrawler(crawler,
extraVars={
'a': 2,
'b': 3
})
self.assertEqual(
os.path.normpath(result),
os.path.normpath(
'/a/b/c/7/v001/RND-TST-SHT_lighting_beauty_sr.001001.exr'))
def testPython(self):
"""
Test that the Python subprocess works properly.
"""
resource = Resource.get()
resource.load(self.__taskPath)
crawler = FsCrawler.createFromPath(self.__sourcePath)
dummyTask = Task.create('pythonTestTask')
dummyTask.add(crawler)
dummyTask.setOption("runPython", False)
wrapper = TaskWrapper.create('python')
result = wrapper.run(dummyTask)
self.assertTrue(len(result), 1)
self.assertIn("testPython", result[0].varNames())
self.assertEqual(result[0].var("testPython"), OpenImageIO.VERSION)