def testRunConfiguration(self): """ Test execution of the configuration. """ loader = Loader() loader.loadFromDirectory(self.__exampleDirectory) self.assertEqual(len(loader.taskHolders()), 1) taskHolder = loader.taskHolders()[0] taskHolder.addVar( "prefix", self.__exampleTargetPrefixDirectory, True ) # loading input data for the execution crawlerGroups = Crawler.group( FsCrawler.createFromPath( os.path.join(self.__exampleDirectory, 'imageSequence') ).globFromParent() ) resultCrawlers = [] for group in crawlerGroups: if isinstance(group[0], Crawler.registeredType('png')): resultCrawlers += taskHolder.run(group) targetFilePaths = list(sorted(filter(lambda x: len(x), map(lambda x: x.strip(), self.__generatedData.split('\n'))))) createdFilePaths = list(sorted(map(lambda x: x.var('fullPath')[len(self.__exampleTargetPrefixDirectory) + 1:].replace('\\', '/'), resultCrawlers))) self.assertListEqual(targetFilePaths, createdFilePaths)
def test02Delivery(self): """ Test the delivery configuration. """ loader = Loader() loader.loadFromDirectory(self.__exampleDirectory) taskHolder = list( filter( lambda x: os.path.basename(x.var('contextConfig')) == 'deliveryConfig.json', loader.taskHolders())) self.assertEqual(len(taskHolder), 1) taskHolder = taskHolder[0] taskHolder.addVar("prefix", self.__exampleDeliveryPrefixDirectory, True) # loading input data for the ingestion crawlerGroups = Crawler.group( FsCrawler.createFromPath( os.path.normpath( os.path.join( self.__exampleIngestionPrefixDirectory, 'jobs/foo/seq/abc/shot/def/plates/bla/v001/1920x1080_exr' ))).glob()) resultCrawlers = [] for group in crawlerGroups: if isinstance(group[0], Crawler.registeredType('plateExr')): resultCrawlers += taskHolder.run(group) targetFilePaths = list( sorted( filter( lambda x: len(x), map( lambda x: x.strip(), self.__deliveryGeneratedData.replace( '<date>', datetime.today().strftime('%Y%m%d')).split( '\n'))))) createdFilePaths = list( sorted( map( lambda x: x.var('fullPath')[len( self.__exampleDeliveryPrefixDirectory) + 1:].replace( '\\', '/'), resultCrawlers))) self.assertListEqual(targetFilePaths, createdFilePaths)
def test01Ingestion(self): """ Test the ingestion configuration. """ loader = Loader() loader.loadFromDirectory(self.__exampleDirectory) taskHolder = list( filter( lambda x: os.path.basename(x.var('contextConfig')) == 'ingestConfig.yaml', loader.taskHolders())) self.assertEqual(len(taskHolder), 1) taskHolder = taskHolder[0] taskHolder.addVar("prefix", self.__exampleIngestionPrefixDirectory, True) # loading input data for the ingestion crawlerGroups = Crawler.group( FsCrawler.createFromPath( os.path.join(self.__exampleDirectory, 'plates')).globFromParent()) resultCrawlers = [] for group in crawlerGroups: if isinstance(group[0], Crawler.registeredType('png')): resultCrawlers += taskHolder.run(group) targetFilePaths = list( sorted( filter( lambda x: len(x), map(lambda x: x.strip(), self.__ingestedGeneratedData.split('\n'))))) createdFilePaths = list( sorted( map( lambda x: x.var('fullPath')[len( self.__exampleIngestionPrefixDirectory) + 1:].replace( '\\', '/'), resultCrawlers))) self.assertListEqual(targetFilePaths, createdFilePaths)
def __groupCrawlers(self, crawlers): """ Return a dictionary containing the matched crawlers grouped. """ groupedCrawlers = OrderedDict() groupedCrawlers[None] = [] for crawlerList in Crawler.group(crawlers): for crawler in crawlerList: # group if self.__checkedViewMode == 'Group' and 'group' in crawler.tagNames( ): groupName = crawler.tag('group') if groupName not in groupedCrawlers: groupedCrawlers[groupName] = [] groupedCrawlers[groupName].append(crawler) # flat else: groupedCrawlers[None].append(crawler) return groupedCrawlers
def __onPerformTasks(self): """ Slot triggered when run button is triggered. """ if not self.__targetTree.model().rowCount(): QtWidgets.QMessageBox.information( self.__main, "Kombi", "No targets available (refresh the targets)!", QtWidgets.QMessageBox.Ok) return visibleCrawlers = self.__visibleCrawlers() # applying overrides self.__applySourceOverrides(self.__loadSourceOverrides(), visibleCrawlers) groupedVisibleCrawlers = Crawler.group(visibleCrawlers) output = '' try: for crawlersGroup in groupedVisibleCrawlers: for taskHolder in self.__taskHolders: # run on the farm if self.__runOnTheFarmCheckbox.checkState( ) == QtCore.Qt.Checked: renderFarmDispatcher = Dispatcher.create('renderFarm') label = "{}/{}/{} [{}]".format( os.path.basename( taskHolder.var('configDirectory')), os.path.splitext( os.path.basename( taskHolder.var('contextConfig')))[0], crawlersGroup[0].tag('group') if 'group' in crawlersGroup[0].tagNames() else crawlersGroup[0].var('baseName'), datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S")) renderFarmDispatcher.setOption('label', label) renderFarmDispatcher.dispatch(taskHolder, crawlersGroup) # run locally else: localDispatcher = Dispatcher.create('local') localDispatcher.setOption('awaitExecution', True) for processExecution in localDispatcher.dispatch( taskHolder, crawlersGroup): output += processExecution.stdoutContent() except Exception as err: traceback.print_exc() messageBox = QtWidgets.QMessageBox(self.__main, "Kombi", QtWidgets.QMessageBox.Ok) messageBox.setIcon(QtWidgets.QMessageBox.Critical) messageBox.setText("Failed during execution") messageBox.setDetailedText(str(err)) messageBox.show() raise err else: message = "Execution completed!" if self.__runOnTheFarmCheckbox.checkState() == QtCore.Qt.Checked: message = "Execution submitted to the farm!" QtWidgets.QMessageBox.information(self.__main, "Kombi", message, QtWidgets.QMessageBox.Ok) # showing the output for local executions if not self.__runOnTheFarmCheckbox.checkState( ) == QtCore.Qt.Checked: self.__outputWidget = QtWidgets.QPlainTextEdit() self.__outputWidget.setPlainText(output) self.__outputWidget.setWindowTitle('Output') self.__outputWidget.setMinimumWidth(920) self.__outputWidget.setMinimumHeight(600) self.__outputWidget.setVisible(True)