def test_upgradeAllNodes():
registerNodeType(SampleNodeV1)
registerNodeType(SampleNodeV2)
g = Graph('')
n1 = g.addNewNode("SampleNodeV1")
n2 = g.addNewNode("SampleNodeV2")
n1Name = n1.name
n2Name = n2.name
graphFile = os.path.join(tempfile.mkdtemp(), "test_description_conflict.mg")
g.save(graphFile)
# make SampleNodeV2 an unknown type
unregisterNodeType(SampleNodeV2)
# replace SampleNodeV1 by SampleNodeV2
meshroom.core.nodesDesc[SampleNodeV1.__name__] = SampleNodeV2
# reload file
g = loadGraph(graphFile)
os.remove(graphFile)
# both nodes are CompatibilityNodes
assert len(g.compatibilityNodes) == 2
assert g.node(n1Name).canUpgrade # description conflict
assert not g.node(n2Name).canUpgrade # unknown type
# upgrade all upgradable nodes
g.upgradeAllNodes()
# only the node with an unknown type has not been upgraded
assert len(g.compatibilityNodes) == 1
assert n2Name in g.compatibilityNodes.keys()
unregisterNodeType(SampleNodeV1)
def test_unknown_node_type():
"""
Test compatibility behavior for unknown node type.
"""
registerNodeType(SampleNodeV1)
g = Graph('')
n = g.addNewNode("SampleNodeV1", input="/dev/null", paramA="foo")
graphFile = os.path.join(tempfile.mkdtemp(), "test_unknown_node_type.mg")
g.save(graphFile)
internalFolder = n.internalFolder
nodeName = n.name
unregisterNodeType(SampleNodeV1)
# reload file
g = loadGraph(graphFile)
os.remove(graphFile)
assert len(g.nodes) == 1
n = g.node(nodeName)
# SampleNodeV1 is now an unknown type
# check node instance type and compatibility issue type
assert isinstance(n, CompatibilityNode)
assert n.issue == CompatibilityIssue.UnknownNodeType
# check if attributes are properly restored
assert len(n.attributes) == 3
assert n.input.isInput
assert n.output.isOutput
# check if internal folder
assert n.internalFolder == internalFolder
# upgrade can't be perform on unknown node types
assert not n.canUpgrade
with pytest.raises(NodeUpgradeError):
g.upgradeNode(nodeName)
def test_conformUpgrade():
registerNodeType(SampleNodeV5)
registerNodeType(SampleNodeV6)
g = Graph('')
n1 = g.addNewNode("SampleNodeV5")
n1.paramA.value = [{
'a':
0,
'b': [{
'a': 0,
'b': [1.0, 2.0]
}, {
'a': 1,
'b': [1.0, 2.0]
}]
}]
n1Name = n1.name
graphFile = os.path.join(tempfile.mkdtemp(), "test_conform_upgrade.mg")
g.save(graphFile)
# replace SampleNodeV5 by SampleNodeV6
meshroom.core.nodesDesc[SampleNodeV5.__name__] = SampleNodeV6
# reload file
g = loadGraph(graphFile)
os.remove(graphFile)
# node is a CompatibilityNode
assert len(g.compatibilityNodes) == 1
assert g.node(n1Name).canUpgrade
# upgrade all upgradable nodes
g.upgradeAllNodes()
# only the node with an unknown type has not been upgraded
assert len(g.compatibilityNodes) == 0
upgradedNode = g.node(n1Name)
# check upgrade
assert isinstance(upgradedNode, Node) and isinstance(
upgradedNode.nodeDesc, SampleNodeV6)
# check conformation
assert len(upgradedNode.paramA.value) == 1
unregisterNodeType(SampleNodeV5)
unregisterNodeType(SampleNodeV6)
def test_description_conflict():
"""
Test compatibility behavior for conflicting node descriptions.
"""
# copy registered node types to be able to restore them
originalNodeTypes = copy.copy(meshroom.core.nodesDesc)
nodeTypes = [SampleNodeV1, SampleNodeV2, SampleNodeV3, SampleNodeV4, SampleNodeV5]
nodes = []
g = Graph('')
# register and instantiate instances of all node types except last one
for nt in nodeTypes[:-1]:
registerNodeType(nt)
n = g.addNewNode(nt.__name__)
if nt == SampleNodeV4:
# initialize list attribute with values to create a conflict with V5
n.paramA.value = [{'a': 0, 'b': [1.0, 2.0]}]
nodes.append(n)
graphFile = os.path.join(tempfile.mkdtemp(), "test_description_conflict.mg")
g.save(graphFile)
# reload file as-is, ensure no compatibility issue is detected (no CompatibilityNode instances)
g = loadGraph(graphFile)
assert all(isinstance(n, Node) for n in g.nodes)
# offset node types register to create description conflicts
# each node type name now reference the next one's implementation
for i, nt in enumerate(nodeTypes[:-1]):
meshroom.core.nodesDesc[nt.__name__] = nodeTypes[i+1]
# reload file
g = loadGraph(graphFile)
os.remove(graphFile)
assert len(g.nodes) == len(nodes)
for srcNode in nodes:
nodeName = srcNode.name
compatNode = g.node(srcNode.name)
# Node description clashes between what has been saved
assert isinstance(compatNode, CompatibilityNode)
assert srcNode.internalFolder == compatNode.internalFolder
# case by case description conflict verification
if isinstance(srcNode.nodeDesc, SampleNodeV1):
# V1 => V2: 'input' has been renamed to 'in'
assert len(compatNode.attributes) == 3
assert hasattr(compatNode, "input")
assert not hasattr(compatNode, "in")
# perform upgrade
upgradedNode = g.upgradeNode(nodeName)[0]
assert isinstance(upgradedNode, Node) and isinstance(upgradedNode.nodeDesc, SampleNodeV2)
assert not hasattr(upgradedNode, "input")
assert hasattr(upgradedNode, "in")
# check uid has changed (not the same set of attributes)
assert upgradedNode.internalFolder != srcNode.internalFolder
elif isinstance(srcNode.nodeDesc, SampleNodeV2):
# V2 => V3: 'paramA' has been removed'
assert len(compatNode.attributes) == 3
assert hasattr(compatNode, "paramA")
# perform upgrade
upgradedNode = g.upgradeNode(nodeName)[0]
assert isinstance(upgradedNode, Node) and isinstance(upgradedNode.nodeDesc, SampleNodeV3)
assert not hasattr(upgradedNode, "paramA")
# check uid is identical (paramA not part of uid)
assert upgradedNode.internalFolder == srcNode.internalFolder
elif isinstance(srcNode.nodeDesc, SampleNodeV3):
# V3 => V4: 'paramA' has been added
assert len(compatNode.attributes) == 2
assert not hasattr(compatNode, "paramA")
# perform upgrade
upgradedNode = g.upgradeNode(nodeName)[0]
assert isinstance(upgradedNode, Node) and isinstance(upgradedNode.nodeDesc, SampleNodeV4)
assert hasattr(upgradedNode, "paramA")
assert isinstance(upgradedNode.paramA.attributeDesc, desc.ListAttribute)
# paramA child attributes invalidate UID
assert upgradedNode.internalFolder != srcNode.internalFolder
elif isinstance(srcNode.nodeDesc, SampleNodeV4):
# V4 => V5: 'paramA' elementDesc has changed from SampleGroupV1 to SampleGroupV2
assert len(compatNode.attributes) == 3
assert hasattr(compatNode, "paramA")
groupAttribute = compatNode.paramA.attributeDesc.elementDesc
assert isinstance(groupAttribute, desc.GroupAttribute)
# check that Compatibility node respect SampleGroupV1 description
for elt in groupAttribute.groupDesc:
assert isinstance(elt, next(a for a in SampleGroupV1 if a.name == elt.name).__class__)
# perform upgrade
upgradedNode = g.upgradeNode(nodeName)[0]
assert isinstance(upgradedNode, Node) and isinstance(upgradedNode.nodeDesc, SampleNodeV5)
assert hasattr(upgradedNode, "paramA")
# parameter was incompatible, value could not be restored
assert upgradedNode.paramA.isDefault
assert upgradedNode.internalFolder != srcNode.internalFolder
else:
raise ValueError("Unexpected node type: " + srcNode.nodeType)
# restore original node types
meshroom.core.nodesDesc = originalNodeTypes
class UIGraph(QObject):
""" High level wrapper over core.Graph, with additional features dedicated to UI integration.
UIGraph exposes undoable methods on its graph and computation in a separate thread.
It also provides a monitoring of all its computation units (NodeChunks).
"""
def __init__(self, parent=None):
super(UIGraph, self).__init__(parent)
self._undoStack = commands.UndoStack(self)
self._graph = Graph('', self)
self._modificationCount = 0
self._chunksMonitor = ChunksMonitor(parent=self)
self._computeThread = Thread()
self._running = self._submitted = False
self._sortedDFSChunks = QObjectListModel(parent=self)
self._layout = GraphLayout(self)
self._selectedNode = None
self._hoveredNode = None
def setGraph(self, g):
""" Set the internal graph. """
if self._graph:
self.stopExecution()
self.clear()
oldGraph = self._graph
self._graph = g
if oldGraph:
oldGraph.deleteLater()
self._graph.updated.connect(self.onGraphUpdated)
self._graph.update()
# perform auto-layout if graph does not provide nodes positions
if Graph.IO.Features.NodesPositions not in self._graph.fileFeatures:
self._layout.reset()
# clear undo-stack after layout
self._undoStack.clear()
else:
bbox = self._layout.positionBoundingBox()
if bbox[2] == 0 and bbox[3] == 0:
self._layout.reset()
# clear undo-stack after layout
self._undoStack.clear()
self.graphChanged.emit()
def onGraphUpdated(self):
""" Callback to any kind of attribute modification. """
# TODO: handle this with a better granularity
self.updateChunks()
def updateChunks(self):
dfsNodes = self._graph.dfsOnFinish(None)[0]
chunks = self._graph.getChunks(dfsNodes)
# Nothing has changed, return
if self._sortedDFSChunks.objectList() == chunks:
return
for chunk in self._sortedDFSChunks:
chunk.statusChanged.disconnect(self.updateGraphComputingStatus)
self._sortedDFSChunks.setObjectList(chunks)
for chunk in self._sortedDFSChunks:
chunk.statusChanged.connect(self.updateGraphComputingStatus)
# provide ChunkMonitor with the update list of chunks
self.updateChunkMonitor(self._sortedDFSChunks)
# update graph computing status based on the new list of NodeChunks
self.updateGraphComputingStatus()
def updateChunkMonitor(self, chunks):
""" Update the list of chunks for status files monitoring. """
self._chunksMonitor.setChunks(chunks)
def clear(self):
if self._graph:
self.clearNodeHover()
self.clearNodeSelection()
self._graph.clear()
self._sortedDFSChunks.clear()
self._undoStack.clear()
def stopChildThreads(self):
""" Stop all child threads. """
self.stopExecution()
self._chunksMonitor.stop()
@Slot(str, result=bool)
def loadGraph(self, filepath, setupProjectFile=True):
g = Graph('')
status = g.load(filepath, setupProjectFile)
if not os.path.exists(g.cacheDir):
os.mkdir(g.cacheDir)
self.setGraph(g)
return status
@Slot(QUrl)
def saveAs(self, url):
if isinstance(url, (str)):
localFile = url
else:
localFile = url.toLocalFile()
# ensure file is saved with ".mg" extension
if os.path.splitext(localFile)[-1] != ".mg":
localFile += ".mg"
self._graph.save(localFile)
self._undoStack.setClean()
# saving file on disk impacts cache folder location
# => force re-evaluation of monitored status files paths
self.updateChunkMonitor(self._sortedDFSChunks)
@Slot()
def save(self):
self._graph.save()
self._undoStack.setClean()
@Slot(Node)
def execute(self, node=None):
if self.computing:
return
nodes = [node] if node else None
self._computeThread = Thread(target=self._execute, args=(nodes, ))
self._computeThread.start()
def _execute(self, nodes):
self.computeStatusChanged.emit()
try:
executeGraph(self._graph, nodes)
except Exception as e:
logging.error("Error during Graph execution {}".format(e))
finally:
self.computeStatusChanged.emit()
@Slot()
def stopExecution(self):
if not self.isComputingLocally():
return
self._graph.stopExecution()
self._computeThread.join()
self.computeStatusChanged.emit()
@Slot(Node)
def submit(self, node=None):
""" Submit the graph to the default Submitter.
If a node is specified, submit this node and its uncomputed predecessors.
Otherwise, submit the whole
Notes:
Default submitter is specified using the MESHROOM_DEFAULT_SUBMITTER environment variable.
"""
self.save() # graph must be saved before being submitted
node = [node] if node else None
submitGraph(self._graph,
os.environ.get('MESHROOM_DEFAULT_SUBMITTER', ''), node)
def updateGraphComputingStatus(self):
# update graph computing status
running = any([
ch.status.status == Status.RUNNING for ch in self._sortedDFSChunks
])
submitted = any([
ch.status.status == Status.SUBMITTED
for ch in self._sortedDFSChunks
])
if self._running != running or self._submitted != submitted:
self._running = running
self._submitted = submitted
self.computeStatusChanged.emit()
def isComputing(self):
""" Whether is graph is being computed, either locally or externally. """
return self.isComputingLocally() or self.isComputingExternally()
def isComputingExternally(self):
""" Whether this graph is being computed externally. """
return (self._running
or self._submitted) and not self.isComputingLocally()
def isComputingLocally(self):
""" Whether this graph is being computed locally (i.e computation can be stopped). """
return self._computeThread.is_alive()
def push(self, command):
""" Try and push the given command to the undo stack.
Args:
command (commands.UndoCommand): the command to push
"""
return self._undoStack.tryAndPush(command)
def groupedGraphModification(self, title, disableUpdates=True):
""" Get a GroupedGraphModification for this Graph.
Args:
title (str): the title of the macro command
disableUpdates (bool): whether to disable graph updates
Returns:
GroupedGraphModification: the instantiated context manager
"""
return commands.GroupedGraphModification(self._graph, self._undoStack,
title, disableUpdates)
def beginModification(self, name):
""" Begin a Graph modification. Calls to beginModification and endModification may be nested, but
every call to beginModification must have a matching call to endModification. """
self._modificationCount += 1
self._undoStack.beginMacro(name)
def endModification(self):
""" Ends a Graph modification. Must match a call to beginModification. """
assert self._modificationCount > 0
self._modificationCount -= 1
self._undoStack.endMacro()
@Slot(str, QPoint, result=QObject)
def addNewNode(self, nodeType, position=None, **kwargs):
""" [Undoable]
Create a new Node of type 'nodeType' and returns it.
Args:
nodeType (str): the type of the Node to create.
position (QPoint): (optional) the initial position of the node
**kwargs: optional node attributes values
Returns:
Node: the created node
"""
if isinstance(position, QPoint):
position = Position(position.x(), position.y())
return self.push(
commands.AddNodeCommand(self._graph,
nodeType,
position=position,
**kwargs))
@Slot(Node, QPoint)
def moveNode(self, node, position):
"""
Move 'node' to the given 'position'.
Args:
node (Node): the node to move
position (QPoint): the target position
"""
if isinstance(position, QPoint):
position = Position(position.x(), position.y())
self.push(commands.MoveNodeCommand(self._graph, node, position))
@Slot(Node)
def removeNode(self, node):
self.push(commands.RemoveNodeCommand(self._graph, node))
@Slot(Node)
def removeNodesFrom(self, startNode):
"""
Remove all nodes starting from 'startNode' to graph leaves.
Args:
startNode (Node): the node to start from.
"""
with self.groupedGraphModification("Remove Nodes from {}".format(
startNode.name)):
# Perform nodes removal from leaves to start node so that edges
# can be re-created in correct order on redo.
[
self.removeNode(node)
for node in reversed(self._graph.nodesFromNode(startNode)[0])
]
@Slot(Attribute, Attribute)
def addEdge(self, src, dst):
if isinstance(dst,
ListAttribute) and not isinstance(src, ListAttribute):
with self.groupedGraphModification(
"Insert and Add Edge on {}".format(dst.getFullName())):
self.appendAttribute(dst)
self.push(commands.AddEdgeCommand(self._graph, src,
dst.at(-1)))
else:
self.push(commands.AddEdgeCommand(self._graph, src, dst))
@Slot(Edge)
def removeEdge(self, edge):
if isinstance(edge.dst.root, ListAttribute):
with self.groupedGraphModification(
"Remove Edge and Delete {}".format(
edge.dst.getFullName())):
self.push(commands.RemoveEdgeCommand(self._graph, edge))
self.removeAttribute(edge.dst)
else:
self.push(commands.RemoveEdgeCommand(self._graph, edge))
@Slot(Attribute, "QVariant")
def setAttribute(self, attribute, value):
self.push(commands.SetAttributeCommand(self._graph, attribute, value))
@Slot(Attribute)
def resetAttribute(self, attribute):
""" Reset 'attribute' to its default value """
self.push(
commands.SetAttributeCommand(self._graph, attribute,
attribute.defaultValue()))
@Slot(Node, bool, result="QVariantList")
def duplicateNode(self, srcNode, duplicateFollowingNodes=False):
"""
Duplicate a node an optionally all the following nodes to graph leaves.
Args:
srcNode (Node): node to start the duplication from
duplicateFollowingNodes (bool): whether to duplicate all the following nodes to graph leaves
Returns:
[Nodes]: the list of duplicated nodes
"""
title = "Duplicate Nodes from {}" if duplicateFollowingNodes else "Duplicate {}"
# enable updates between duplication and layout to get correct depths during layout
with self.groupedGraphModification(title.format(srcNode.name),
disableUpdates=False):
# disable graph updates during duplication
with self.groupedGraphModification("Node duplication",
disableUpdates=True):
duplicates = self.push(
commands.DuplicateNodeCommand(self._graph, srcNode,
duplicateFollowingNodes))
# move nodes below the bounding box formed by the duplicated node(s)
bbox = self._layout.boundingBox(duplicates)
for n in duplicates:
self.moveNode(
n, Position(n.x, bbox[3] + self.layout.gridSpacing + n.y))
return duplicates
@Slot(CompatibilityNode, result=Node)
def upgradeNode(self, node):
""" Upgrade a CompatibilityNode. """
return self.push(commands.UpgradeNodeCommand(self._graph, node))
@Slot()
def upgradeAllNodes(self):
""" Upgrade all upgradable CompatibilityNode instances in the graph. """
with self.groupedGraphModification("Upgrade all Nodes"):
nodes = [
n for n in self._graph._compatibilityNodes.values()
if n.canUpgrade
]
for node in nodes:
self.upgradeNode(node)
@Slot()
def forceNodesStatusUpdate(self):
""" Force re-evaluation of graph's nodes status. """
self._graph.updateStatusFromCache(force=True)
@Slot(Attribute, QJsonValue)
def appendAttribute(self, attribute, value=QJsonValue()):
if isinstance(value, QJsonValue):
if value.isArray():
pyValue = value.toArray().toVariantList()
else:
pyValue = None if value.isNull() else value.toObject()
else:
pyValue = value
self.push(
commands.ListAttributeAppendCommand(self._graph, attribute,
pyValue))
@Slot(Attribute)
def removeAttribute(self, attribute):
self.push(commands.ListAttributeRemoveCommand(self._graph, attribute))
def clearNodeSelection(self):
""" Clear node selection. """
self.selectedNode = None
def clearNodeHover(self):
""" Reset currently hovered node to None. """
self.hoveredNode = None
undoStack = Property(QObject, lambda self: self._undoStack, constant=True)
graphChanged = Signal()
graph = Property(Graph, lambda self: self._graph, notify=graphChanged)
nodes = Property(QObject,
lambda self: self._graph.nodes,
notify=graphChanged)
layout = Property(GraphLayout, lambda self: self._layout, constant=True)
computeStatusChanged = Signal()
computing = Property(bool, isComputing, notify=computeStatusChanged)
computingExternally = Property(bool,
isComputingExternally,
notify=computeStatusChanged)
computingLocally = Property(bool,
isComputingLocally,
notify=computeStatusChanged)
canSubmit = Property(bool, lambda self: len(submitters), constant=True)
sortedDFSChunks = Property(QObject,
lambda self: self._sortedDFSChunks,
constant=True)
lockedChanged = Signal()
selectedNodeChanged = Signal()
# Currently selected node
selectedNode = makeProperty(QObject,
"_selectedNode",
selectedNodeChanged,
resetOnDestroy=True)
hoveredNodeChanged = Signal()
# Currently hovered node
hoveredNode = makeProperty(QObject,
"_hoveredNode",
hoveredNodeChanged,
resetOnDestroy=True)
