def test_upgradeAllNodes(): registerNodeType(SampleNodeV1) registerNodeType(SampleNodeV2) g = Graph('') n1 = g.addNewNode("SampleNodeV1") n2 = g.addNewNode("SampleNodeV2") n1Name = n1.name n2Name = n2.name graphFile = os.path.join(tempfile.mkdtemp(), "test_description_conflict.mg") g.save(graphFile) # make SampleNodeV2 an unknown type unregisterNodeType(SampleNodeV2) # replace SampleNodeV1 by SampleNodeV2 meshroom.core.nodesDesc[SampleNodeV1.__name__] = SampleNodeV2 # reload file g = loadGraph(graphFile) os.remove(graphFile) # both nodes are CompatibilityNodes assert len(g.compatibilityNodes) == 2 assert g.node(n1Name).canUpgrade # description conflict assert not g.node(n2Name).canUpgrade # unknown type # upgrade all upgradable nodes g.upgradeAllNodes() # only the node with an unknown type has not been upgraded assert len(g.compatibilityNodes) == 1 assert n2Name in g.compatibilityNodes.keys() unregisterNodeType(SampleNodeV1)
def test_unknown_node_type(): """ Test compatibility behavior for unknown node type. """ registerNodeType(SampleNodeV1) g = Graph('') n = g.addNewNode("SampleNodeV1", input="/dev/null", paramA="foo") graphFile = os.path.join(tempfile.mkdtemp(), "test_unknown_node_type.mg") g.save(graphFile) internalFolder = n.internalFolder nodeName = n.name unregisterNodeType(SampleNodeV1) # reload file g = loadGraph(graphFile) os.remove(graphFile) assert len(g.nodes) == 1 n = g.node(nodeName) # SampleNodeV1 is now an unknown type # check node instance type and compatibility issue type assert isinstance(n, CompatibilityNode) assert n.issue == CompatibilityIssue.UnknownNodeType # check if attributes are properly restored assert len(n.attributes) == 3 assert n.input.isInput assert n.output.isOutput # check if internal folder assert n.internalFolder == internalFolder # upgrade can't be perform on unknown node types assert not n.canUpgrade with pytest.raises(NodeUpgradeError): g.upgradeNode(nodeName)
def test_conformUpgrade(): registerNodeType(SampleNodeV5) registerNodeType(SampleNodeV6) g = Graph('') n1 = g.addNewNode("SampleNodeV5") n1.paramA.value = [{ 'a': 0, 'b': [{ 'a': 0, 'b': [1.0, 2.0] }, { 'a': 1, 'b': [1.0, 2.0] }] }] n1Name = n1.name graphFile = os.path.join(tempfile.mkdtemp(), "test_conform_upgrade.mg") g.save(graphFile) # replace SampleNodeV5 by SampleNodeV6 meshroom.core.nodesDesc[SampleNodeV5.__name__] = SampleNodeV6 # reload file g = loadGraph(graphFile) os.remove(graphFile) # node is a CompatibilityNode assert len(g.compatibilityNodes) == 1 assert g.node(n1Name).canUpgrade # upgrade all upgradable nodes g.upgradeAllNodes() # only the node with an unknown type has not been upgraded assert len(g.compatibilityNodes) == 0 upgradedNode = g.node(n1Name) # check upgrade assert isinstance(upgradedNode, Node) and isinstance( upgradedNode.nodeDesc, SampleNodeV6) # check conformation assert len(upgradedNode.paramA.value) == 1 unregisterNodeType(SampleNodeV5) unregisterNodeType(SampleNodeV6)
def test_description_conflict(): """ Test compatibility behavior for conflicting node descriptions. """ # copy registered node types to be able to restore them originalNodeTypes = copy.copy(meshroom.core.nodesDesc) nodeTypes = [SampleNodeV1, SampleNodeV2, SampleNodeV3, SampleNodeV4, SampleNodeV5] nodes = [] g = Graph('') # register and instantiate instances of all node types except last one for nt in nodeTypes[:-1]: registerNodeType(nt) n = g.addNewNode(nt.__name__) if nt == SampleNodeV4: # initialize list attribute with values to create a conflict with V5 n.paramA.value = [{'a': 0, 'b': [1.0, 2.0]}] nodes.append(n) graphFile = os.path.join(tempfile.mkdtemp(), "test_description_conflict.mg") g.save(graphFile) # reload file as-is, ensure no compatibility issue is detected (no CompatibilityNode instances) g = loadGraph(graphFile) assert all(isinstance(n, Node) for n in g.nodes) # offset node types register to create description conflicts # each node type name now reference the next one's implementation for i, nt in enumerate(nodeTypes[:-1]): meshroom.core.nodesDesc[nt.__name__] = nodeTypes[i+1] # reload file g = loadGraph(graphFile) os.remove(graphFile) assert len(g.nodes) == len(nodes) for srcNode in nodes: nodeName = srcNode.name compatNode = g.node(srcNode.name) # Node description clashes between what has been saved assert isinstance(compatNode, CompatibilityNode) assert srcNode.internalFolder == compatNode.internalFolder # case by case description conflict verification if isinstance(srcNode.nodeDesc, SampleNodeV1): # V1 => V2: 'input' has been renamed to 'in' assert len(compatNode.attributes) == 3 assert hasattr(compatNode, "input") assert not hasattr(compatNode, "in") # perform upgrade upgradedNode = g.upgradeNode(nodeName)[0] assert isinstance(upgradedNode, Node) and isinstance(upgradedNode.nodeDesc, SampleNodeV2) assert not hasattr(upgradedNode, "input") assert hasattr(upgradedNode, "in") # check uid has changed (not the same set of attributes) assert upgradedNode.internalFolder != srcNode.internalFolder elif isinstance(srcNode.nodeDesc, SampleNodeV2): # V2 => V3: 'paramA' has been removed' assert len(compatNode.attributes) == 3 assert hasattr(compatNode, "paramA") # perform upgrade upgradedNode = g.upgradeNode(nodeName)[0] assert isinstance(upgradedNode, Node) and isinstance(upgradedNode.nodeDesc, SampleNodeV3) assert not hasattr(upgradedNode, "paramA") # check uid is identical (paramA not part of uid) assert upgradedNode.internalFolder == srcNode.internalFolder elif isinstance(srcNode.nodeDesc, SampleNodeV3): # V3 => V4: 'paramA' has been added assert len(compatNode.attributes) == 2 assert not hasattr(compatNode, "paramA") # perform upgrade upgradedNode = g.upgradeNode(nodeName)[0] assert isinstance(upgradedNode, Node) and isinstance(upgradedNode.nodeDesc, SampleNodeV4) assert hasattr(upgradedNode, "paramA") assert isinstance(upgradedNode.paramA.attributeDesc, desc.ListAttribute) # paramA child attributes invalidate UID assert upgradedNode.internalFolder != srcNode.internalFolder elif isinstance(srcNode.nodeDesc, SampleNodeV4): # V4 => V5: 'paramA' elementDesc has changed from SampleGroupV1 to SampleGroupV2 assert len(compatNode.attributes) == 3 assert hasattr(compatNode, "paramA") groupAttribute = compatNode.paramA.attributeDesc.elementDesc assert isinstance(groupAttribute, desc.GroupAttribute) # check that Compatibility node respect SampleGroupV1 description for elt in groupAttribute.groupDesc: assert isinstance(elt, next(a for a in SampleGroupV1 if a.name == elt.name).__class__) # perform upgrade upgradedNode = g.upgradeNode(nodeName)[0] assert isinstance(upgradedNode, Node) and isinstance(upgradedNode.nodeDesc, SampleNodeV5) assert hasattr(upgradedNode, "paramA") # parameter was incompatible, value could not be restored assert upgradedNode.paramA.isDefault assert upgradedNode.internalFolder != srcNode.internalFolder else: raise ValueError("Unexpected node type: " + srcNode.nodeType) # restore original node types meshroom.core.nodesDesc = originalNodeTypes
class UIGraph(QObject): """ High level wrapper over core.Graph, with additional features dedicated to UI integration. UIGraph exposes undoable methods on its graph and computation in a separate thread. It also provides a monitoring of all its computation units (NodeChunks). """ def __init__(self, parent=None): super(UIGraph, self).__init__(parent) self._undoStack = commands.UndoStack(self) self._graph = Graph('', self) self._modificationCount = 0 self._chunksMonitor = ChunksMonitor(parent=self) self._computeThread = Thread() self._running = self._submitted = False self._sortedDFSChunks = QObjectListModel(parent=self) self._layout = GraphLayout(self) self._selectedNode = None self._hoveredNode = None def setGraph(self, g): """ Set the internal graph. """ if self._graph: self.stopExecution() self.clear() oldGraph = self._graph self._graph = g if oldGraph: oldGraph.deleteLater() self._graph.updated.connect(self.onGraphUpdated) self._graph.update() # perform auto-layout if graph does not provide nodes positions if Graph.IO.Features.NodesPositions not in self._graph.fileFeatures: self._layout.reset() # clear undo-stack after layout self._undoStack.clear() else: bbox = self._layout.positionBoundingBox() if bbox[2] == 0 and bbox[3] == 0: self._layout.reset() # clear undo-stack after layout self._undoStack.clear() self.graphChanged.emit() def onGraphUpdated(self): """ Callback to any kind of attribute modification. """ # TODO: handle this with a better granularity self.updateChunks() def updateChunks(self): dfsNodes = self._graph.dfsOnFinish(None)[0] chunks = self._graph.getChunks(dfsNodes) # Nothing has changed, return if self._sortedDFSChunks.objectList() == chunks: return for chunk in self._sortedDFSChunks: chunk.statusChanged.disconnect(self.updateGraphComputingStatus) self._sortedDFSChunks.setObjectList(chunks) for chunk in self._sortedDFSChunks: chunk.statusChanged.connect(self.updateGraphComputingStatus) # provide ChunkMonitor with the update list of chunks self.updateChunkMonitor(self._sortedDFSChunks) # update graph computing status based on the new list of NodeChunks self.updateGraphComputingStatus() def updateChunkMonitor(self, chunks): """ Update the list of chunks for status files monitoring. """ self._chunksMonitor.setChunks(chunks) def clear(self): if self._graph: self.clearNodeHover() self.clearNodeSelection() self._graph.clear() self._sortedDFSChunks.clear() self._undoStack.clear() def stopChildThreads(self): """ Stop all child threads. """ self.stopExecution() self._chunksMonitor.stop() @Slot(str, result=bool) def loadGraph(self, filepath, setupProjectFile=True): g = Graph('') status = g.load(filepath, setupProjectFile) if not os.path.exists(g.cacheDir): os.mkdir(g.cacheDir) self.setGraph(g) return status @Slot(QUrl) def saveAs(self, url): if isinstance(url, (str)): localFile = url else: localFile = url.toLocalFile() # ensure file is saved with ".mg" extension if os.path.splitext(localFile)[-1] != ".mg": localFile += ".mg" self._graph.save(localFile) self._undoStack.setClean() # saving file on disk impacts cache folder location # => force re-evaluation of monitored status files paths self.updateChunkMonitor(self._sortedDFSChunks) @Slot() def save(self): self._graph.save() self._undoStack.setClean() @Slot(Node) def execute(self, node=None): if self.computing: return nodes = [node] if node else None self._computeThread = Thread(target=self._execute, args=(nodes, )) self._computeThread.start() def _execute(self, nodes): self.computeStatusChanged.emit() try: executeGraph(self._graph, nodes) except Exception as e: logging.error("Error during Graph execution {}".format(e)) finally: self.computeStatusChanged.emit() @Slot() def stopExecution(self): if not self.isComputingLocally(): return self._graph.stopExecution() self._computeThread.join() self.computeStatusChanged.emit() @Slot(Node) def submit(self, node=None): """ Submit the graph to the default Submitter. If a node is specified, submit this node and its uncomputed predecessors. Otherwise, submit the whole Notes: Default submitter is specified using the MESHROOM_DEFAULT_SUBMITTER environment variable. """ self.save() # graph must be saved before being submitted node = [node] if node else None submitGraph(self._graph, os.environ.get('MESHROOM_DEFAULT_SUBMITTER', ''), node) def updateGraphComputingStatus(self): # update graph computing status running = any([ ch.status.status == Status.RUNNING for ch in self._sortedDFSChunks ]) submitted = any([ ch.status.status == Status.SUBMITTED for ch in self._sortedDFSChunks ]) if self._running != running or self._submitted != submitted: self._running = running self._submitted = submitted self.computeStatusChanged.emit() def isComputing(self): """ Whether is graph is being computed, either locally or externally. """ return self.isComputingLocally() or self.isComputingExternally() def isComputingExternally(self): """ Whether this graph is being computed externally. """ return (self._running or self._submitted) and not self.isComputingLocally() def isComputingLocally(self): """ Whether this graph is being computed locally (i.e computation can be stopped). """ return self._computeThread.is_alive() def push(self, command): """ Try and push the given command to the undo stack. Args: command (commands.UndoCommand): the command to push """ return self._undoStack.tryAndPush(command) def groupedGraphModification(self, title, disableUpdates=True): """ Get a GroupedGraphModification for this Graph. Args: title (str): the title of the macro command disableUpdates (bool): whether to disable graph updates Returns: GroupedGraphModification: the instantiated context manager """ return commands.GroupedGraphModification(self._graph, self._undoStack, title, disableUpdates) def beginModification(self, name): """ Begin a Graph modification. Calls to beginModification and endModification may be nested, but every call to beginModification must have a matching call to endModification. """ self._modificationCount += 1 self._undoStack.beginMacro(name) def endModification(self): """ Ends a Graph modification. Must match a call to beginModification. """ assert self._modificationCount > 0 self._modificationCount -= 1 self._undoStack.endMacro() @Slot(str, QPoint, result=QObject) def addNewNode(self, nodeType, position=None, **kwargs): """ [Undoable] Create a new Node of type 'nodeType' and returns it. Args: nodeType (str): the type of the Node to create. position (QPoint): (optional) the initial position of the node **kwargs: optional node attributes values Returns: Node: the created node """ if isinstance(position, QPoint): position = Position(position.x(), position.y()) return self.push( commands.AddNodeCommand(self._graph, nodeType, position=position, **kwargs)) @Slot(Node, QPoint) def moveNode(self, node, position): """ Move 'node' to the given 'position'. Args: node (Node): the node to move position (QPoint): the target position """ if isinstance(position, QPoint): position = Position(position.x(), position.y()) self.push(commands.MoveNodeCommand(self._graph, node, position)) @Slot(Node) def removeNode(self, node): self.push(commands.RemoveNodeCommand(self._graph, node)) @Slot(Node) def removeNodesFrom(self, startNode): """ Remove all nodes starting from 'startNode' to graph leaves. Args: startNode (Node): the node to start from. """ with self.groupedGraphModification("Remove Nodes from {}".format( startNode.name)): # Perform nodes removal from leaves to start node so that edges # can be re-created in correct order on redo. [ self.removeNode(node) for node in reversed(self._graph.nodesFromNode(startNode)[0]) ] @Slot(Attribute, Attribute) def addEdge(self, src, dst): if isinstance(dst, ListAttribute) and not isinstance(src, ListAttribute): with self.groupedGraphModification( "Insert and Add Edge on {}".format(dst.getFullName())): self.appendAttribute(dst) self.push(commands.AddEdgeCommand(self._graph, src, dst.at(-1))) else: self.push(commands.AddEdgeCommand(self._graph, src, dst)) @Slot(Edge) def removeEdge(self, edge): if isinstance(edge.dst.root, ListAttribute): with self.groupedGraphModification( "Remove Edge and Delete {}".format( edge.dst.getFullName())): self.push(commands.RemoveEdgeCommand(self._graph, edge)) self.removeAttribute(edge.dst) else: self.push(commands.RemoveEdgeCommand(self._graph, edge)) @Slot(Attribute, "QVariant") def setAttribute(self, attribute, value): self.push(commands.SetAttributeCommand(self._graph, attribute, value)) @Slot(Attribute) def resetAttribute(self, attribute): """ Reset 'attribute' to its default value """ self.push( commands.SetAttributeCommand(self._graph, attribute, attribute.defaultValue())) @Slot(Node, bool, result="QVariantList") def duplicateNode(self, srcNode, duplicateFollowingNodes=False): """ Duplicate a node an optionally all the following nodes to graph leaves. Args: srcNode (Node): node to start the duplication from duplicateFollowingNodes (bool): whether to duplicate all the following nodes to graph leaves Returns: [Nodes]: the list of duplicated nodes """ title = "Duplicate Nodes from {}" if duplicateFollowingNodes else "Duplicate {}" # enable updates between duplication and layout to get correct depths during layout with self.groupedGraphModification(title.format(srcNode.name), disableUpdates=False): # disable graph updates during duplication with self.groupedGraphModification("Node duplication", disableUpdates=True): duplicates = self.push( commands.DuplicateNodeCommand(self._graph, srcNode, duplicateFollowingNodes)) # move nodes below the bounding box formed by the duplicated node(s) bbox = self._layout.boundingBox(duplicates) for n in duplicates: self.moveNode( n, Position(n.x, bbox[3] + self.layout.gridSpacing + n.y)) return duplicates @Slot(CompatibilityNode, result=Node) def upgradeNode(self, node): """ Upgrade a CompatibilityNode. """ return self.push(commands.UpgradeNodeCommand(self._graph, node)) @Slot() def upgradeAllNodes(self): """ Upgrade all upgradable CompatibilityNode instances in the graph. """ with self.groupedGraphModification("Upgrade all Nodes"): nodes = [ n for n in self._graph._compatibilityNodes.values() if n.canUpgrade ] for node in nodes: self.upgradeNode(node) @Slot() def forceNodesStatusUpdate(self): """ Force re-evaluation of graph's nodes status. """ self._graph.updateStatusFromCache(force=True) @Slot(Attribute, QJsonValue) def appendAttribute(self, attribute, value=QJsonValue()): if isinstance(value, QJsonValue): if value.isArray(): pyValue = value.toArray().toVariantList() else: pyValue = None if value.isNull() else value.toObject() else: pyValue = value self.push( commands.ListAttributeAppendCommand(self._graph, attribute, pyValue)) @Slot(Attribute) def removeAttribute(self, attribute): self.push(commands.ListAttributeRemoveCommand(self._graph, attribute)) def clearNodeSelection(self): """ Clear node selection. """ self.selectedNode = None def clearNodeHover(self): """ Reset currently hovered node to None. """ self.hoveredNode = None undoStack = Property(QObject, lambda self: self._undoStack, constant=True) graphChanged = Signal() graph = Property(Graph, lambda self: self._graph, notify=graphChanged) nodes = Property(QObject, lambda self: self._graph.nodes, notify=graphChanged) layout = Property(GraphLayout, lambda self: self._layout, constant=True) computeStatusChanged = Signal() computing = Property(bool, isComputing, notify=computeStatusChanged) computingExternally = Property(bool, isComputingExternally, notify=computeStatusChanged) computingLocally = Property(bool, isComputingLocally, notify=computeStatusChanged) canSubmit = Property(bool, lambda self: len(submitters), constant=True) sortedDFSChunks = Property(QObject, lambda self: self._sortedDFSChunks, constant=True) lockedChanged = Signal() selectedNodeChanged = Signal() # Currently selected node selectedNode = makeProperty(QObject, "_selectedNode", selectedNodeChanged, resetOnDestroy=True) hoveredNodeChanged = Signal() # Currently hovered node hoveredNode = makeProperty(QObject, "_hoveredNode", hoveredNodeChanged, resetOnDestroy=True)