Пример #1
0
 def _RDFGraph(self):
     # expensive to recompute
     graph = Graph()
     for URL, obj in self._pypeObjects.iteritems():
         for s,p,o in obj._RDFGraph:
             graph.add( (s,p,o) )
     return graph
Пример #2
0
class GrowingTree(object):
    def __init__(self, width, height, strategy):
        self.width = width
        self.height = height
        self.strategy = strategy
        self.graph = Graph(width, height)

    def run(self, verbose=False):
        cells = [(randrange(self.width), randrange(self.height)),]
        while len(cells) > 0:
            if verbose:
                os.system('clear')
                print self.graph.__unicode__()
                time.sleep(0.01)
            index = self.strategy(len(cells))
            x, y = cells[index]
            dirs = [D.N, D.S, D.E, D.W]
            shuffle(dirs)
            for dir in dirs:
                # find neighbour
                nx, ny = x + D.DX[dir], y + D.DY[dir]
                # in bounds and neighbour unvisited?
                if 0 <= nx < self.width and 0 <= ny < self.height and self.graph[ny,nx] == 0:
                    self.graph[y,x] |= dir
                    self.graph[ny][nx] |= D.OPP[dir]
                    index = -1
                    cells.append((nx, ny),)
            if index >= 0:
                cells.pop(index)
Пример #3
0
 def _RDFGraph(self):
     # expensive to recompute
     graph = Graph()
     for URL, obj in self._pypeObjects.iteritems():
         for s, p, o in obj._RDFGraph:
             graph.add((s, p, o))
     return graph
Пример #4
0
 def setReferenceRDFGraph(self, fn):
     self._referenceRDFGraph = Graph()
     self._referenceRDFGraph.load(fn)
     refMD5s = self._referenceRDFGraph.subject_objects(
         pypeNS["codeMD5digest"])
     for URL, md5digest in refMD5s:
         obj = self._pypeObjects[str(URL)]
         obj.setReferenceMD5(md5digest)
Пример #5
0
def main(args):
    with io.open(args.file) as f:
        edges = list(parse_graph(f))
    g = Graph(edges)
    scores = g.page_rank()
    print(">" * 80)
    for node, score in sorted(scores.items(), key=lambda x: x[1],
                              reverse=True):
        print("{} ({})".format(node, score))
    print("<" * 80)
Пример #6
0
 def test_my2(self):
     edges = [
         ["A", "B"],
         ["B", "C"],
         ["C", "A"],
         ["C", "B"],
         ["C", "D"],
     ]
     g = Graph(edges)
     # scores se espera que sea {nodo1: score1, nodo2: score2, ...}
     scores = g.page_rank(damping=0.85, limit=1.0e-8)
     sorted_nodes = [node for node, _ in sorted(scores.items(), key=lambda x: x[1], reverse=True)]
     self.assertSequenceEqual(sorted_nodes, ["C", "B", "A", "D"])
Пример #7
0
 def setReferenceRDFGraph(self, fn):
     self._referenceRDFGraph = Graph()
     self._referenceRDFGraph.load(fn)
     refMD5s = self._referenceRDFGraph.subject_objects(pypeNS["codeMD5digest"])
     for URL, md5digest in refMD5s:
         obj = self._pypeObjects[str(URL)]
         obj.setReferenceMD5(md5digest)
Пример #8
0
def depthFirstSearch(graph: com.Graph, currentNode: str, endNode: str,
                     visited: Set[str], currentPath: List[str],
                     allPaths: list):
    if currentNode in visited:
        return
    if not isBigCave(currentNode):
        visited.add(currentNode)
    currentPath.append(currentNode)
    if currentNode == endNode:
        allPaths.append(currentPath)
        return
    edges = graph.direct_connected_weights_and_edges(currentNode)
    for node in edges:
        if node not in visited:
            depthFirstSearch(graph, node, endNode, set(visited),
                             list(currentPath), allPaths)
Пример #9
0
def depthFirstSearchAllowRevisited(graph: com.Graph, currentNode: str,
                                   endNode: str, visited: Set[str],
                                   visitedTwice: Set[str],
                                   currentPath: List[str], allPaths: list):
    if not canVisit(currentNode, visited, visitedTwice, True):
        return

    currentPath.append(currentNode)

    if currentNode == endNode:
        allPaths.append(currentPath)
        return
    edges = graph.direct_connected_weights_and_edges(currentNode)
    for node in edges:
        if canVisit(node, visited, visitedTwice):
            depthFirstSearchAllowRevisited(graph, node, endNode, set(visited),
                                           set(visitedTwice),
                                           list(currentPath), allPaths)
Пример #10
0
def findShortest(graph: com.Graph, currentNode, ownedKeys: set, allKeys: set, currentSteps: int):
    pathQueue = deque()
    alreadyHit = {}
    alreadyHit[(currentNode, frozenset(ownedKeys))] = currentSteps
    keysInOrder = list()
    pathQueue.append((currentSteps,currentNode, ownedKeys, keysInOrder))
    allKeysHash = frozenset(allKeys).__hash__()
    shortestSoFar = impossible
    shortestPath = None
    while pathQueue:
        currentSteps, currentNode, ownedKeys, keysInOrder = pathQueue.popleft()
        ownedKeys = ownedKeys.copy()
        keysInOrder = keysInOrder.copy()
        if isKey(currentNode):
            ownedKeys.add(currentNode)
            keysInOrder.extend(currentNode)

        currentOwnedKeys = frozenset(ownedKeys)
        if allKeysHash == currentOwnedKeys.__hash__():
            if shortestSoFar >= currentSteps:
                shortestSoFar = currentSteps
                shortestPath = keysInOrder
                continue
            else:
                continue

        for weight, edge in graph.direct_connected_weights_and_edges(currentNode):
            newSteps = currentSteps + weight
            if isDoor(edge) and edge.lower() not in ownedKeys:
                continue
            if alreadyHit.get((edge, currentOwnedKeys)):
                if alreadyHit.get((edge, currentOwnedKeys)) < newSteps:
                    continue
            alreadyHit[(edge, currentOwnedKeys)] = newSteps
            pathQueue.append((newSteps, edge, ownedKeys, keysInOrder))
    return shortestSoFar, shortestPath
Пример #11
0
# -*- coding:utf-8 -*-
"""
BGM1.py: Bipartite Graph Matching with 1-star local structure

Written by Ding Rui
Latest Version: 2020/5/24
"""

from common import ArgParse, Graph, CostMatrix, PrintGED, Add1Star, SolveLSAP

dot_sub, dot_ins, dot_del, edge_sub, edge_ins, edge_del, root_path, void, inf,\
    g1, g2 = ArgParse()

g1 = Graph(root_path + '/' + g1)
g2 = Graph(root_path + '/' + g2)

_, col, _ = SolveLSAP(CostMatrix(g1, g2, Add1Star))
answer = [(int(col[i]) if col[i] < g2.dots else void) for i in range(g1.dots)]
PrintGED(g1, g2, tuple(answer))
Пример #12
0
#graph.add_edges( 4, 15, 9)
#graph.add_edges( 5, 14, 11)
#graph.add_edges( 6, 15, 9, 1, 12, 7, 3)
#graph.add_edges( 7, 6, 3, 10, 8, 13, 12)
#graph.add_edges( 8, 2, 10, 7, 13)
#graph.add_edges( 9, 4, 15, 6, 1)
#graph.add_edges(10, 3, 7, 8, 2)
#graph.add_edges(11, 5, 14, 12, 13)
#graph.add_edges(12, 1, 6, 7, 13, 11, 14)
#graph.add_edges(13, 11, 12, 7, 8)
#graph.add_edges(14, 5, 11, 12, 1)
#graph.add_edges(15, 4, 9, 6, 3)

if __name__ == '__main__':
    total = 40
    graph = Graph(nodes=range(1, 16))
    graph.add_edges( 1, )
    graph.add_edges( 2, )
    graph.add_edges( 3, )
    graph.add_edges( 4, )
    graph.add_edges( 5, )
    graph.add_edges( 6, )
    graph.add_edges( 7, )
    graph.add_edges( 8, )
    graph.add_edges( 9, )
    graph.add_edges(10, )
    graph.add_edges(11, )
    graph.add_edges(12, )
    graph.add_edges(13, )
    graph.add_edges(14, )
    graph.add_edges(15, )
Пример #13
0
 def __init__(self, width, height, strategy):
     self.width = width
     self.height = height
     self.strategy = strategy
     self.graph = Graph(width, height)
Пример #14
0
def solve():
    matrix = [map(int, line.strip().split(',')) for line in open('resources/p081_matrix.txt')]
    
    g = Graph()
    g.add_node('source')
    g.add_node('target')
    
    for r in xrange(len(matrix)):
        for c in xrange(len(matrix[r])):
            g.add_node((r, c))
            
    for r in xrange(len(matrix)):
        for c in xrange(len(matrix[r])):
            if r > 0:
                g.add_edge((r-1,c), (r,c), matrix[r][c])
            if c > 0:
                g.add_edge((r,c-1), (r,c), matrix[r][c])
    g.add_edge('source', (0, 0), matrix[0][0])
    g.add_edge((len(matrix)-1, len(matrix[0])-1), 'target', 0)
    
    costs, _ = dijkstra(g, 'source')

    return costs['target']
Пример #15
0
    def parse_models(self, parse_stateful = False):
        """Method to query and read data from database.

        Method to query database and read models into Graph objects.

        Args:
            parse_stateful (bool) : Boolean to indicate whether graphs with 
                stateful partitioned call should be parsed, these graphs do not
                contain a graph structure or tensors. Defaults to False.

        Returns:
            List of Graph objects corresponding to the graph objects the models
            in the spanner database have been parsed into.
        """

        model_graphs = list()
        
        # Query to get all models from Models table
        with self.database.snapshot() as snapshot:
            qresult_models = snapshot.execute_sql(
                "SELECT model_name, category, sub_category, source, num_inputs"
                " FROM Models"
                )

        for row in qresult_models:

            # Checking num_inputs for presence of graph structure
            if row[4] == 0 and not parse_stateful:
                continue

            # Extracting model attributes
            model_name = row[0]
            category = row[1]
            sub_category = row[2]
            source = row[3]

            nodes = list()
            edges = list()
            start_node_indices = list()

            adj_list = dict()

            # Querying Operators of model_name
            with self.database.snapshot() as snapshot:
                qresult_operators = snapshot.execute_sql(
                    "SELECT * from Models JOIN Operators"
                    " ON Models.model_name = Operators.model_name"
                    " WHERE Models.model_name = '" + model_name + "'"
                    " ORDER BY operator_id"
                )
            
            # Dictionary to hold which field is in which index of query results
            field_to_index = dict()

            # Boolean to check if field_to_dict needs to be populated
            populate_dicts = True

            # Extracting Node attributes
            for row in qresult_operators:
                if populate_dicts:
                    for index in range(len(qresult_operators.metadata.row_type.fields)):
                        field_name = qresult_operators.metadata.row_type.fields[index].name
                        field_to_index[field_name] = index
                    
                    populate_dicts = False

                new_node = Node.Node(None, None)

                for attr in vars(new_node).keys():
                    if attr in field_to_index:
                        setattr(new_node, attr, row[field_to_index[attr]])

                nodes.append(new_node)

                # populating start_node_indices using is_input field
                if row[field_to_index['is_input']]:
                    start_node_indices.append(len(nodes) - 1)
            
            # Querying Tensors of model_name
            with self.database.snapshot() as snapshot:
                qresult_tensors = snapshot.execute_sql(
                    "SELECT * from Models JOIN Tensors"
                    " ON Models.model_name = Tensors.model_name"
                    " WHERE Models.model_name = '" + model_name + "'"
                    " ORDER BY tensor_id"
                )

            # Dictionary to hold which field is in which index of query results
            field_to_index.clear()

            # Boolean to check if field_to_dict needs to be populated
            populate_dicts = True

            # Extracting Edge attributes
            for row in qresult_tensors:
                if populate_dicts:
                    for index in range(len(qresult_tensors.metadata.row_type.fields)):
                        field_name = qresult_tensors.metadata.row_type.fields[index].name
                        field_to_index[field_name] = index
                    
                    populate_dicts = False

                new_edge = Edge.Edge(None, None)

                for attr in vars(new_edge).keys():
                    if attr in field_to_index:
                        setattr(new_edge, attr, row[field_to_index[attr]])

                edges.append(new_edge)

                to_operator_ids = row[field_to_index['to_operator_ids']]
                from_operator_ids = row[field_to_index['from_operator_ids']]

                edge_index = len(edges) - 1

                for src_node_index in from_operator_ids:
                    src_node_index -= 1
                    for dest_node_index in to_operator_ids:
                        dest_node_index -= 1

                        if src_node_index not in adj_list:
                            adj_list.update({src_node_index : []})
                        
                        adj_list[src_node_index].append([edge_index, 
                                                            dest_node_index])

            new_graph = Graph.Graph(nodes, start_node_indices, edges, adj_list, 
                                    model_name, category, sub_category)
            new_graph.source = source

            model_graphs.append(new_graph)

        return model_graphs
Пример #16
0
class PypeWorkflow(PypeObject):
    """ 
    Representing a PypeWorkflow. PypeTask and PypeDataObjects can be added
    into the workflow and executed through the instanct methods.

    >>> import os, time 
    >>> from pypeflow.data import PypeLocalFile, makePypeLocalFile, fn
    >>> from pypeflow.task import *
    >>> try:
    ...     os.makedirs("/tmp/pypetest")
    ...     _ = os.system("rm -f /tmp/pypetest/*")
    ... except Exception:
    ...     pass
    >>> time.sleep(1)
    >>> fin = makePypeLocalFile("/tmp/pypetest/testfile_in", readOnly=False)
    >>> fout = makePypeLocalFile("/tmp/pypetest/testfile_out", readOnly=False)
    >>> @PypeTask(outputDataObjs={"test_out":fout},
    ...           inputDataObjs={"test_in":fin},
    ...           parameters={"a":'I am "a"'}, **{"b":'I am "b"'})
    ... def test(self):
    ...     print test.test_in.localFileName
    ...     print test.test_out.localFileName
    ...     os.system( "touch %s" % fn(test.test_out) )
    ...     pass
    >>> os.system( "touch %s" %  (fn(fin))  )
    0
    >>> from pypeflow.controller import PypeWorkflow
    >>> wf = PypeWorkflow()
    >>> wf.addTask(test)
    >>> def finalize(self):
    ...     def f():
    ...         print "in finalize:", self._status
    ...     return f
    >>> test.finalize = finalize(test)  # For testing only. Please don't do this in your code. The PypeTask.finalized() is intended to be overriden by subclasses. 
    >>> wf.refreshTargets( objs = [fout] )
    /tmp/pypetest/testfile_in
    /tmp/pypetest/testfile_out
    in finalize: done
    True
    """

    supportedURLScheme = ["workflow"]

    def __init__(self, URL = None, **attributes ):

        if URL == None:
            URL = "workflow://" + __file__+"/%d" % id(self)

        self._pypeObjects = {}

        PypeObject.__init__(self, URL, **attributes)

        self._referenceRDFGraph = None #place holder for a reference RDF

        
    def addObject(self, obj):
        self.addObjects([obj])

    def addObjects(self, objs):
        """
        Add data objects into the workflow. One can add also task object to the workflow using this method for
        non-threaded workflow.
        """
        for obj in objs:
            if obj.URL in self._pypeObjects:
                if id(self._pypeObjects[obj.URL]) != id(obj):
                    raise PypeError, "Add different objects with the same URL %s" % obj.URL
                else:
                    continue
            self._pypeObjects[obj.URL] = obj

    def addTask(self, taskObj):
        self.addTasks([taskObj])


    def addTasks(self, taskObjs):
        """
        Add tasks into the workflow. The dependent input and output data objects are added automatically too. 
        It sets the message queue used for communicating between the task thread and the main thread. One has
        to use addTasks() or addTask() to add task objects to a threaded workflow.
        """
        for taskObj in taskObjs:
            if isinstance(taskObj, PypeTaskCollection):
                for subTaskObj in taskObj.getTasks() + taskObj.getScatterGatherTasks():
                    self.addObjects(subTaskObj.inputDataObjs.values())
                    self.addObjects(subTaskObj.outputDataObjs.values())
                    self.addObjects(subTaskObj.mutableDataObjs.values())
                    self.addObject(subTaskObj)

            else:
                for dObj in taskObj.inputDataObjs.values() +\
                            taskObj.outputDataObjs.values() +\
                            taskObj.mutableDataObjs.values() :
                    if isinstance(dObj, PypeSplittableLocalFile):
                        self.addObjects([dObj._completeFile])
                    self.addObjects([dObj])

                self.addObject(taskObj)

            
    def removeTask(self, taskObj):
        self.removeTasks([taskObj])
        
    def removeTasks(self, taskObjs ):
        """
        Remove tasks from the workflow.
        """
        self.removeObjects(taskObjs)
            
    def removeObjects(self, objs):
        """
        Remove objects from the workflow. If the object cannot be found, a PypeError is raised.
        """
        for obj in objs:
            if obj.URL in self._pypeObjects:
                del self._pypeObjects[obj.URL]
            else:
                raise PypeError, "Unable to remove %s from the graph. (Object not found)" % obj.URL

    def updateURL(self, oldURL, newURL):
        obj = self._pypeObjects[oldURL]
        obj._updateURL(newURL)
        self._pypeObjects[newURL] = obj
        del self._pypeObjects[oldURL]


            
    @property
    def _RDFGraph(self):
        # expensive to recompute
        graph = Graph()
        for URL, obj in self._pypeObjects.iteritems():
            for s,p,o in obj._RDFGraph:
                graph.add( (s,p,o) )
        return graph

    def setReferenceRDFGraph(self, fn):
        self._referenceRDFGraph = Graph()
        self._referenceRDFGraph.load(fn)
        refMD5s = self._referenceRDFGraph.subject_objects(pypeNS["codeMD5digest"])
        for URL, md5digest in refMD5s:
            obj = self._pypeObjects[str(URL)]
            obj.setReferenceMD5(md5digest)

    def _graphvizDot(self, shortName=False):
        graph = self._RDFGraph
        dotStr = StringIO()
        shapeMap = {"file":"box", "state":"box", "task":"component"}
        colorMap = {"file":"yellow", "state":"cyan", "task":"green"}
        dotStr.write( 'digraph "%s" {\n rankdir=LR;' % self.URL)
        for URL in self._pypeObjects.keys():
            URLParseResult = urlparse(URL)
            if URLParseResult.scheme not in shapeMap:
                continue
            else:
                shape = shapeMap[URLParseResult.scheme]
                color = colorMap[URLParseResult.scheme]

                s = URL
                if shortName == True:
                    s = URLParseResult.scheme + "://..." + URLParseResult.path.split("/")[-1] 
                dotStr.write( '"%s" [shape=%s, fillcolor=%s, style=filled];\n' % (s, shape, color))

        for row in graph.query('SELECT ?s ?o WHERE {?s pype:prereq ?o . }', initNs=dict(pype=pypeNS)):
            s, o = row
            if shortName == True:
                    s = urlparse(s).scheme + "://..." + urlparse(s).path.split("/")[-1] 
                    o = urlparse(o).scheme + "://..." + urlparse(o).path.split("/")[-1] 
            dotStr.write( '"%s" -> "%s";\n' % (o, s))
        for row in graph.query('SELECT ?s ?o WHERE {?s pype:hasMutable ?o . }', initNs=dict(pype=pypeNS)):
            s, o = row
            if shortName == True:
                    s = urlparse(s).scheme + "://..." + urlparse(s).path.split("/")[-1] 
                    o = urlparse(o).scheme + "://..." + urlparse(o).path.split("/")[-1] 
            dotStr.write( '"%s" -- "%s" [arrowhead=both, style=dashed ];\n' % (s, o))
        dotStr.write ("}")
        return dotStr.getvalue()

    @property
    def graphvizDot(self):
        return self._graphvizDot()

    @property
    def graphvizShortNameDot(self):
        return self._graphvizDot(shortName = True)

    @property
    def makeFileStr(self):
        """
        generate a string that has the information of the execution dependency in
        a "Makefile" like format. It can be written into a "Makefile" and
        executed by "make".
        """
        for URL in self._pypeObjects.keys():
            URLParseResult = urlparse(URL)
            if URLParseResult.scheme != "task": continue
            taskObj = self._pypeObjects[URL]
            if not hasattr(taskObj, "script"):
                raise TaskTypeError("can not convert non shell script based workflow to a makefile") 
        makeStr = StringIO()
        for URL in self._pypeObjects.keys():
            URLParseResult = urlparse(URL)
            if URLParseResult.scheme != "task": continue
            taskObj = self._pypeObjects[URL]
            inputFiles = taskObj.inputDataObjs
            outputFiles = taskObj.outputDataObjs
            #for oStr in [o.localFileName for o in outputFiles.values()]:
            if 1:
                oStr = " ".join( [o.localFileName for o in outputFiles.values()])

                iStr = " ".join([i.localFileName for i in inputFiles.values()])
                makeStr.write( "%s:%s\n" % ( oStr, iStr ) )
                makeStr.write( "\t%s\n\n" % taskObj.script )
        makeStr.write("all: %s" %  " ".join([o.localFileName for o in outputFiles.values()]) )
        return makeStr.getvalue()

    @staticmethod
    def getSortedURLs(rdfGraph, objs):
        if len(objs) != 0:
            connectedPypeNodes = set()
            for obj in objs:
                if isinstance(obj, PypeSplittableLocalFile):
                    obj = obj._completeFile
                for x in rdfGraph.transitive_objects(URIRef(obj.URL), pypeNS["prereq"]):
                    connectedPypeNodes.add(x)
            tSortedURLs = PypeGraph(rdfGraph, connectedPypeNodes).tSort( )
        else:
            tSortedURLs = PypeGraph(rdfGraph).tSort( )
        return tSortedURLs

    def refreshTargets(self, objs = [], callback = (None, None, None) ):
        """
        Execute the DAG to reach all objects in the "objs" argument.
        """
        tSortedURLs = self.getSortedURLs(self._RDFGraph, objs)
        for URL in tSortedURLs:
            obj = self._pypeObjects[URL]
            if not isinstance(obj, PypeTaskBase):
                continue
            else:
                obj()
                obj.finalize()
        self._runCallback(callback)
        return True

    def _runCallback(self, callback = (None, None, None ) ):
        if callback[0] != None and callable(callback[0]):
            argv = []
            kwargv = {}
            if callback[1] != None and isinstance( callback[1], type(list()) ):
                argv = callback[1]
            else:
                raise TaskExecutionError( "callback argument type error") 

            if callback[2] != None and isinstance( callback[1], type(dict()) ):
                kwargv = callback[2]
            else:
                raise TaskExecutionError( "callback argument type error") 

            callback[0](*argv, **kwargv)

        elif callback[0] != None:
            raise TaskExecutionError( "callback is not callable") 
    
    @property
    def dataObjects( self ):
        return [ o for o in self._pypeObjects.values( ) if isinstance( o, PypeDataObjectBase )]
    
    @property
    def tasks( self ):
        return [ o for o in self._pypeObjects.values( ) if isinstance( o, PypeTaskBase )]

    @property
    def inputDataObjects(self):
        graph = self._RDFGraph
        inputObjs = []
        for obj in self.dataObjects:
            r = graph.query('SELECT ?o WHERE {<%s> pype:prereq ?o .  }' % obj.URL, initNs=dict(pype=pypeNS))
            if len(r) == 0:
                inputObjs.append(obj)
        return inputObjs
     
    @property
    def outputDataObjects(self):
        graph = self._RDFGraph
        outputObjs = []
        for obj in self.dataObjects:
            r = graph.query('SELECT ?s WHERE {?s pype:prereq <%s> .  }' % obj.URL, initNs=dict(pype=pypeNS))
            if len(r) == 0:
                outputObjs.append(obj)
        return outputObjs
Пример #17
0
                    stack.append((edge, iter(graph.get_edges(edge))))
                    visitor.send((Event.TreeEdge, node, edge))
                    visitor.send((Event.DiscoverVertex, edge))
                elif color[edge] == Color.Gray:
                    visitor.send((Event.BackEdge, node, edge))

            except StopIteration:
                node, _ = stack.pop()
                color[node] = Color.Black
                visitor.send((Event.FinishVertex, node))

if __name__ == "__main__":
    from common import Graph
    def print_visitor(graph):
        while True:
            message = yield
            print Event.get_name(message[0]),
            if Event.is_vertex_event(message[0]):
                print "\t[{}]".format(message[1])
            else: print "\t[{}]->[{}]".format(message[1], message[2])

    graph = Graph(['a', 'b', 'c', 'd', 'e', 'f'])
    graph.add_edges('a', 'b', 'c')
    graph.add_edges('b', 'c', 'e')
    graph.add_edges('c', 'd', 'f')
    graph.add_edges('d', 'e')
    graph.add_edges('e', 'f')
    visitor = print_visitor(graph)
    visitor.next()
    graph_dfs_visit(graph, 'a', visitor)
Пример #18
0
        edges1 = 0
        edges2 = 0
        for val in dict1.values():
            if val > 0:
                edges1 += val
            else:
                edges2 -= val
        # 边替换总不劣于边删除+边插入
        if edges1 > edges2:
            result += (edges2 * edge_sub + (edges1 - edges2) * edge_del)
        else:
            result += (edges1 * edge_sub + (edges2 - edges1) * edge_ins)
        return result


graph1 = Graph(root_path + '/'+ graph1)
graph2 = Graph(root_path + '/'+ graph2)

queue = PriorityQueue()
for i in range(graph2.dots):
    queue.put(Partial(graph1, graph2, 0, tuple(), i))
queue.put(Partial(graph1, graph2, 0, tuple(), void))

while True:
    partial = queue.get()
    if len(partial.part_map) == graph1.dots:
        PrintGED(graph1, graph2, partial.part_map)
        break
    else:
        left = set(range(graph2.dots)) - set(partial.part_map)
        for i in left:
Пример #19
0
    def _RDFGraph(self):
        graph = Graph()
        for k,v in self.__dict__.iteritems():
            if k == "URL": continue
            if k[0] == "_": continue
            if k in ["inputDataObjs", "outputDataObjs", "mutableDataObjs", "parameters"]:
                if k == "inputDataObjs":
                    for ft, f in v.iteritems():
                        graph.add( (URIRef(self.URL), pypeNS["prereq"], URIRef(f.URL) ) )
                elif k == "outputDataObjs":
                    for ft, f in v.iteritems():
                        graph.add( (URIRef(f.URL), pypeNS["prereq"], URIRef(self.URL) ) )
                elif k == "mutableDataObjs":
                    for ft, f in v.iteritems():
                        graph.add( (URIRef(self.URL), pypeNS["hasMutable"], URIRef(f.URL)   ) )
                elif k == "parameters":
                    graph.add( (URIRef(self.URL), pypeNS["hasParameters"], Literal(json.dumps(v)) ) )
            
                continue

            if k in self.inputDataObjs:
                graph.add( ( URIRef(self.URL), pypeNS["inputDataObject"], URIRef(v.URL) ) )
                continue

            if k in self.outputDataObjs:
                graph.add( ( URIRef(self.URL), pypeNS["outputDataObject"], URIRef(v.URL) ) )
                continue

            if k in self.mutableDataObjs:
                graph.add( ( URIRef(self.URL), pypeNS["mutableDataObject"], URIRef(v.URL) ) )
                continue

            if hasattr(v, "URL"):
                graph.add( ( URIRef(self.URL), pypeNS[k], URIRef(v.URL) ) )

            graph.add(  ( URIRef(self.URL), pypeNS["codeMD5digest"], Literal(self._codeMD5digest) ) )
            graph.add(  ( URIRef(self.URL), pypeNS["parameterMD5digest"], Literal(self._paramMD5digest) ) )

        return graph
Пример #20
0
class PypeWorkflow(PypeObject):
    """ 
    Representing a PypeWorkflow. PypeTask and PypeDataObjects can be added
    into the workflow and executed through the instanct methods.

    >>> import os, time 
    >>> from pypeflow.data import PypeLocalFile, makePypeLocalFile, fn
    >>> from pypeflow.task import *
    >>> try:
    ...     os.makedirs("/tmp/pypetest")
    ...     _ = os.system("rm -f /tmp/pypetest/*")
    ... except Exception:
    ...     pass
    >>> time.sleep(1)
    >>> fin = makePypeLocalFile("/tmp/pypetest/testfile_in", readOnly=False)
    >>> fout = makePypeLocalFile("/tmp/pypetest/testfile_out", readOnly=False)
    >>> @PypeTask(outputDataObjs={"test_out":fout},
    ...           inputDataObjs={"test_in":fin},
    ...           parameters={"a":'I am "a"'}, **{"b":'I am "b"'})
    ... def test(self):
    ...     print test.test_in.localFileName
    ...     print test.test_out.localFileName
    ...     os.system( "touch %s" % fn(test.test_out) )
    ...     pass
    >>> os.system( "touch %s" %  (fn(fin))  )
    0
    >>> from pypeflow.controller import PypeWorkflow
    >>> wf = PypeWorkflow()
    >>> wf.addTask(test)
    >>> def finalize(self):
    ...     def f():
    ...         print "in finalize:", self._status
    ...     return f
    >>> test.finalize = finalize(test)  # For testing only. Please don't do this in your code. The PypeTask.finalized() is intended to be overriden by subclasses. 
    >>> wf.refreshTargets( objs = [fout] )
    /tmp/pypetest/testfile_in
    /tmp/pypetest/testfile_out
    in finalize: done
    True
    """

    supportedURLScheme = ["workflow"]

    def __init__(self, URL=None, **attributes):

        if URL == None:
            URL = "workflow://" + __file__ + "/%d" % id(self)

        self._pypeObjects = {}

        PypeObject.__init__(self, URL, **attributes)

        self._referenceRDFGraph = None  #place holder for a reference RDF

    def addObject(self, obj):
        self.addObjects([obj])

    def addObjects(self, objs):
        """
        Add data objects into the workflow. One can add also task object to the workflow using this method for
        non-threaded workflow.
        """
        for obj in objs:
            if obj.URL in self._pypeObjects:
                if id(self._pypeObjects[obj.URL]) != id(obj):
                    raise PypeError, "Add different objects with the same URL %s" % obj.URL
                else:
                    continue
            self._pypeObjects[obj.URL] = obj

    def addTask(self, taskObj):
        self.addTasks([taskObj])

    def addTasks(self, taskObjs):
        """
        Add tasks into the workflow. The dependent input and output data objects are added automatically too. 
        It sets the message queue used for communicating between the task thread and the main thread. One has
        to use addTasks() or addTask() to add task objects to a threaded workflow.
        """
        for taskObj in taskObjs:
            if isinstance(taskObj, PypeTaskCollection):
                for subTaskObj in taskObj.getTasks(
                ) + taskObj.getScatterGatherTasks():
                    self.addObjects(subTaskObj.inputDataObjs.values())
                    self.addObjects(subTaskObj.outputDataObjs.values())
                    self.addObjects(subTaskObj.mutableDataObjs.values())
                    self.addObject(subTaskObj)

            else:
                for dObj in taskObj.inputDataObjs.values() +\
                            taskObj.outputDataObjs.values() +\
                            taskObj.mutableDataObjs.values() :
                    if isinstance(dObj, PypeSplittableLocalFile):
                        self.addObjects([dObj._completeFile])
                    self.addObjects([dObj])

                self.addObject(taskObj)

    def removeTask(self, taskObj):
        self.removeTasks([taskObj])

    def removeTasks(self, taskObjs):
        """
        Remove tasks from the workflow.
        """
        self.removeObjects(taskObjs)

    def removeObjects(self, objs):
        """
        Remove objects from the workflow. If the object cannot be found, a PypeError is raised.
        """
        for obj in objs:
            if obj.URL in self._pypeObjects:
                del self._pypeObjects[obj.URL]
            else:
                raise PypeError, "Unable to remove %s from the graph. (Object not found)" % obj.URL

    def updateURL(self, oldURL, newURL):
        obj = self._pypeObjects[oldURL]
        obj._updateURL(newURL)
        self._pypeObjects[newURL] = obj
        del self._pypeObjects[oldURL]

    @property
    def _RDFGraph(self):
        # expensive to recompute
        graph = Graph()
        for URL, obj in self._pypeObjects.iteritems():
            for s, p, o in obj._RDFGraph:
                graph.add((s, p, o))
        return graph

    def setReferenceRDFGraph(self, fn):
        self._referenceRDFGraph = Graph()
        self._referenceRDFGraph.load(fn)
        refMD5s = self._referenceRDFGraph.subject_objects(
            pypeNS["codeMD5digest"])
        for URL, md5digest in refMD5s:
            obj = self._pypeObjects[str(URL)]
            obj.setReferenceMD5(md5digest)

    def _graphvizDot(self, shortName=False):
        graph = self._RDFGraph
        dotStr = StringIO()
        shapeMap = {"file": "box", "state": "box", "task": "component"}
        colorMap = {"file": "yellow", "state": "cyan", "task": "green"}
        dotStr.write('digraph "%s" {\n rankdir=LR;' % self.URL)
        for URL in self._pypeObjects.keys():
            URLParseResult = urlparse(URL)
            if URLParseResult.scheme not in shapeMap:
                continue
            else:
                shape = shapeMap[URLParseResult.scheme]
                color = colorMap[URLParseResult.scheme]

                s = URL
                if shortName == True:
                    s = URLParseResult.scheme + "://..." + URLParseResult.path.split(
                        "/")[-1]
                dotStr.write('"%s" [shape=%s, fillcolor=%s, style=filled];\n' %
                             (s, shape, color))

        for row in graph.query('SELECT ?s ?o WHERE {?s pype:prereq ?o . }',
                               initNs=dict(pype=pypeNS)):
            s, o = row
            if shortName == True:
                s = urlparse(s).scheme + "://..." + urlparse(s).path.split(
                    "/")[-1]
                o = urlparse(o).scheme + "://..." + urlparse(o).path.split(
                    "/")[-1]
            dotStr.write('"%s" -> "%s";\n' % (o, s))
        for row in graph.query('SELECT ?s ?o WHERE {?s pype:hasMutable ?o . }',
                               initNs=dict(pype=pypeNS)):
            s, o = row
            if shortName == True:
                s = urlparse(s).scheme + "://..." + urlparse(s).path.split(
                    "/")[-1]
                o = urlparse(o).scheme + "://..." + urlparse(o).path.split(
                    "/")[-1]
            dotStr.write('"%s" -- "%s" [arrowhead=both, style=dashed ];\n' %
                         (s, o))
        dotStr.write("}")
        return dotStr.getvalue()

    @property
    def graphvizDot(self):
        return self._graphvizDot()

    @property
    def graphvizShortNameDot(self):
        return self._graphvizDot(shortName=True)

    @property
    def makeFileStr(self):
        """
        generate a string that has the information of the execution dependency in
        a "Makefile" like format. It can be written into a "Makefile" and
        executed by "make".
        """
        for URL in self._pypeObjects.keys():
            URLParseResult = urlparse(URL)
            if URLParseResult.scheme != "task": continue
            taskObj = self._pypeObjects[URL]
            if not hasattr(taskObj, "script"):
                raise TaskTypeError(
                    "can not convert non shell script based workflow to a makefile"
                )
        makeStr = StringIO()
        for URL in self._pypeObjects.keys():
            URLParseResult = urlparse(URL)
            if URLParseResult.scheme != "task": continue
            taskObj = self._pypeObjects[URL]
            inputFiles = taskObj.inputDataObjs
            outputFiles = taskObj.outputDataObjs
            #for oStr in [o.localFileName for o in outputFiles.values()]:
            if 1:
                oStr = " ".join(
                    [o.localFileName for o in outputFiles.values()])

                iStr = " ".join([i.localFileName for i in inputFiles.values()])
                makeStr.write("%s:%s\n" % (oStr, iStr))
                makeStr.write("\t%s\n\n" % taskObj.script)
        makeStr.write("all: %s" %
                      " ".join([o.localFileName
                                for o in outputFiles.values()]))
        return makeStr.getvalue()

    @staticmethod
    def getSortedURLs(rdfGraph, objs):
        if len(objs) != 0:
            connectedPypeNodes = set()
            for obj in objs:
                if isinstance(obj, PypeSplittableLocalFile):
                    obj = obj._completeFile
                for x in rdfGraph.transitive_objects(URIRef(obj.URL),
                                                     pypeNS["prereq"]):
                    connectedPypeNodes.add(x)
            tSortedURLs = PypeGraph(rdfGraph, connectedPypeNodes).tSort()
        else:
            tSortedURLs = PypeGraph(rdfGraph).tSort()
        return tSortedURLs

    def refreshTargets(self, objs=[], callback=(None, None, None)):
        """
        Execute the DAG to reach all objects in the "objs" argument.
        """
        tSortedURLs = self.getSortedURLs(self._RDFGraph, objs)
        for URL in tSortedURLs:
            obj = self._pypeObjects[URL]
            if not isinstance(obj, PypeTaskBase):
                continue
            else:
                obj()
                obj.finalize()
        self._runCallback(callback)
        return True

    def _runCallback(self, callback=(None, None, None)):
        if callback[0] != None and callable(callback[0]):
            argv = []
            kwargv = {}
            if callback[1] != None and isinstance(callback[1], type(list())):
                argv = callback[1]
            else:
                raise TaskExecutionError("callback argument type error")

            if callback[2] != None and isinstance(callback[1], type(dict())):
                kwargv = callback[2]
            else:
                raise TaskExecutionError("callback argument type error")

            callback[0](*argv, **kwargv)

        elif callback[0] != None:
            raise TaskExecutionError("callback is not callable")

    @property
    def dataObjects(self):
        return [
            o for o in self._pypeObjects.values()
            if isinstance(o, PypeDataObjectBase)
        ]

    @property
    def tasks(self):
        return [
            o for o in self._pypeObjects.values()
            if isinstance(o, PypeTaskBase)
        ]

    @property
    def inputDataObjects(self):
        graph = self._RDFGraph
        inputObjs = []
        for obj in self.dataObjects:
            r = graph.query('SELECT ?o WHERE {<%s> pype:prereq ?o .  }' %
                            obj.URL,
                            initNs=dict(pype=pypeNS))
            if len(r) == 0:
                inputObjs.append(obj)
        return inputObjs

    @property
    def outputDataObjects(self):
        graph = self._RDFGraph
        outputObjs = []
        for obj in self.dataObjects:
            r = graph.query('SELECT ?s WHERE {?s pype:prereq <%s> .  }' %
                            obj.URL,
                            initNs=dict(pype=pypeNS))
            if len(r) == 0:
                outputObjs.append(obj)
        return outputObjs
Пример #21
0
    """
    distance = {n: -sys.maxint for n in graph.nodes()}
    queue, distance[graph.head] = [graph.head], 0
    while any(queue):
        head = queue.pop()
        queue = list(graph.siblings(head)) + queue
        if not any(graph.siblings(head)):
            continue
        for sibling in graph.siblings(head):
            path = distance[head] + graph.edge(head, sibling)
            distance[sibling] = max(distance[sibling], path)
    return distance


# ------------------------------------------------------------
# longest/shortest paths tests
# ------------------------------------------------------------
if __name__ == "__main__":
    graph = Graph("s")
    graph.add_edge("s", "a", 1)
    graph.add_edge("s", "c", 2)
    graph.add_edge("c", "d", 3)
    graph.add_edge("c", "a", 4)
    graph.add_edge("a", "b", 6)
    graph.add_edge("b", "d", 1)
    graph.add_edge("b", "e", 2)
    graph.add_edge("d", "e", 1)

    print shortest_paths(graph)
    print longest_paths(graph)
Пример #22
0
    def _RDFGraph(self):
        graph = Graph()
        for k, v in self.__dict__.iteritems():
            if k == "URL": continue
            if k[0] == "_": continue
            if k in [
                    "inputDataObjs", "outputDataObjs", "mutableDataObjs",
                    "parameters"
            ]:
                if k == "inputDataObjs":
                    for ft, f in v.iteritems():
                        graph.add((URIRef(self.URL), pypeNS["prereq"],
                                   URIRef(f.URL)))
                elif k == "outputDataObjs":
                    for ft, f in v.iteritems():
                        graph.add((URIRef(f.URL), pypeNS["prereq"],
                                   URIRef(self.URL)))
                elif k == "mutableDataObjs":
                    for ft, f in v.iteritems():
                        graph.add((URIRef(self.URL), pypeNS["hasMutable"],
                                   URIRef(f.URL)))
                elif k == "parameters":
                    graph.add((URIRef(self.URL), pypeNS["hasParameters"],
                               Literal(json.dumps(v))))

                continue

            if k in self.inputDataObjs:
                graph.add((URIRef(self.URL), pypeNS["inputDataObject"],
                           URIRef(v.URL)))
                continue

            if k in self.outputDataObjs:
                graph.add((URIRef(self.URL), pypeNS["outputDataObject"],
                           URIRef(v.URL)))
                continue

            if k in self.mutableDataObjs:
                graph.add((URIRef(self.URL), pypeNS["mutableDataObject"],
                           URIRef(v.URL)))
                continue

            if hasattr(v, "URL"):
                graph.add((URIRef(self.URL), pypeNS[k], URIRef(v.URL)))

            graph.add((URIRef(self.URL), pypeNS["codeMD5digest"],
                       Literal(self._codeMD5digest)))
            graph.add((URIRef(self.URL), pypeNS["parameterMD5digest"],
                       Literal(self._paramMD5digest)))

        return graph
Пример #23
0
    def parse_graph(self, file_path, model_name, category, sub_category):
        """Method to parse file and Create a corresponding Graph object.

        Reads a tflite file into a tflite/Model Object and then extracts 
        operators, tensors, graph structure and metadata and stores it 
        into a Graph, Node and Edge objects. Nodes are operations and 
        edges are tensors.

        Args:
            file_path (str): Path of the file to parse
            model_name (str): Unique model name of the model being parsed.
            category (str): Problem category of the model.
            sub_category (str) : Problem sub category of the model.

        Returns:
            The Graph object created for the file.
        """

        model = self.parse(file_path)

        nodes = list()
        edges = list()
        adj_list = dict()
        start_node_indices = list()

        # Global list of opcodes in the model, referenced by Operators
        opcodes = list()
        for opcode_index in range(model.OperatorCodesLength()):
            opcodes.append(model.OperatorCodes(opcode_index))

        # Only considering the main model
        subgraph = model.Subgraphs(0)

        # Dictionary to store origin and destination nodes for each edge
        to_nodes = dict()
        from_nodes = dict()

        for tensor_index in range(subgraph.TensorsLength()):
            tensor = subgraph.Tensors(tensor_index)
            # Converting tensor to an Edge object
            new_edge = self._TENSOR_TO_EDGE.convert(tensor)
            edges.append(new_edge)

        # Populating to_nodes, from_nodes
        # Add proxy nodes for Input and Output of the model
        for input_index in range(subgraph.InputsLength()):
            new_node = Node.Node(label="Input_Placeholder",
                                 operator_type="Input_Placeholder")
            nodes.append(new_node)

            node_index = len(nodes) - 1
            start_node_indices.append(node_index)
            edge_index = subgraph.Inputs(input_index)

            if edge_index not in from_nodes:
                from_nodes.update({edge_index: []})
            from_nodes[edge_index].append(node_index)

        for operator_index in range(subgraph.OperatorsLength()):
            operator = subgraph.Operators(operator_index)
            builtin_opcode = opcodes[operator.OpcodeIndex()].BuiltinCode()
            opname = self._builtin_optype[builtin_opcode]

            new_node = self._OP_TO_NODE.convert(operator, opname)

            # Condition to extract Conv 2D filter sizes and
            # input and output channels as it is contained in tensors
            # and not in operators
            if new_node.label == "CONV_2D":
                weight_tensor = subgraph.Tensors(operator.Inputs(1))
                new_node.filter_height = weight_tensor.Shape(1)
                new_node.filter_width = weight_tensor.Shape(2)

            nodes.append(new_node)
            node_index = len(nodes) - 1

            for input_index in range(operator.InputsLength()):
                edge_index = operator.Inputs(input_index)
                if edge_index not in to_nodes:
                    to_nodes.update({edge_index: list()})

                to_nodes[edge_index].append(node_index)

            for output_index in range(operator.OutputsLength()):
                edge_index = operator.Outputs(output_index)
                if edge_index not in from_nodes:
                    from_nodes.update({edge_index: list()})

                from_nodes[edge_index].append(node_index)

        for output_index in range(subgraph.OutputsLength()):
            new_node = Node.Node(label="Output_Placeholder",
                                 operator_type="Output_Placeholder")
            nodes.append(new_node)

            node_index = len(nodes) - 1
            edge_index = subgraph.Outputs(output_index)

            if edge_index not in to_nodes:
                to_nodes.update({edge_index: []})
            to_nodes[edge_index].append(node_index)

        # Constructing adjacency List from to_nodes, from_nodes
        for edge_index in range(len(edges)):

            if edge_index not in from_nodes or edge_index not in to_nodes:
                continue

            for node1_index in from_nodes[edge_index]:
                for node2_index in to_nodes[edge_index]:
                    if node1_index not in adj_list:
                        adj_list.update({node1_index: list()})

                    adj_list[node1_index].append([edge_index, node2_index])

        graph = Graph.Graph(nodes, start_node_indices, edges, adj_list,
                            model_name, category, sub_category)

        # Removing nodes which are not reachable from input
        graph.process_nodes()
        graph.source = "TFLite"

        return graph
    def parse_graph(self, file_path, model_name, category, sub_category,
                    is_saved_model, input_operation_names):
        """Method to parse file and Create a corresponding Graph object.

        Reads a GraphDef from SavedModel or FrozenGraph file and extracts 
        operations, tensors, graph structure and metadata and stores it 
        into a Graph, Node and Edge objects. Nodes are operations 
        and edges are tensors.

        If graph contains a 'StatefulPartitionedCall' operation,
        all operations are extracted and pushed into the database without tensor
        information or graph structure.

        Args:
            file_path (str): Path of the file to parse
            model_name (str): Unique model name of the model being parsed.
            category (str): Problem category of the model.
            sub_category (str) : Problem sub category of the model.
            is_saved_model (str, optional): "True" if file is in SavedModel format, 
                defaults to "True".
            input_operation_names (list of str, optional) : Names of the operations 
                that are inputs to the model, defaults to [].

        Returns:
            The Graph object created for the file.
        """

        if is_saved_model == "True":
            saved_model = tf.core.protobuf.saved_model_pb2.SavedModel()
            with tf.io.gfile.GFile(file_path, "rb") as f:
                saved_model.ParseFromString(f.read())

            meta_graph = saved_model.meta_graphs[0]
            graph_def = meta_graph.graph_def

        else:
            with tf.io.gfile.GFile(file_path, "rb") as f:
                graph_def = tf.compat.v1.GraphDef()
                graph_def.ParseFromString(f.read())

        with tf.Graph().as_default() as graph:
            tf.import_graph_def(graph_def, name="")

            # Dictionary to store origin and destination nodes for each edge
            to_nodes = dict()
            from_nodes = dict()

            edges = list()
            nodes = list()
            start_node_indices = list()

            tensor_to_index = dict()

            # Loop to populate to_nodes and from_nodes
            for operation in graph.get_operations():

                # If graph contains StatefulPartitionedCall operation,
                # only extracting the operations and returning empty graph
                if operation.node_def.op == "StatefulPartitionedCall":
                    print(
                        "Graphs with operation 'StatefulPartitionedCall' are "
                        "not fully supported for parsing, graph or tensor "
                        "information not supported, only operators will be "
                        "loaded into database.")

                    NODES_DISCARDED = [
                        "Const", "VarHandleOp", "StatefulPartitionedCall",
                        "NoOp", "Identity"
                    ]  # List of operations to not be considered, not of semantic use.

                    nodes.clear()

                    # Looping over all ops in the graph
                    for node_def in graph_def.node:
                        op = node_def.op
                        if op in NODES_DISCARDED or "VariableOp" in op:
                            continue
                        new_node = self._OP_TO_NODE.convert(None, node_def)
                        nodes.append(new_node)

                    # Looping over operations that occur within functions
                    for func in graph_def.library.function:
                        for node_def in func.node_def:
                            op = node_def.op
                            if op in NODES_DISCARDED or "VariableOp" in op:
                                continue

                            new_node = self._OP_TO_NODE.convert(None, node_def)
                            nodes.append(new_node)

                    # Discarding unwanted nodes
                    for index, node in enumerate(nodes):
                        if (node.operator_type in NODES_DISCARDED
                                or "VariableOp" in node.operator_type):
                            nodes.pop(index)

                    new_graph = Graph.Graph(nodes, [], [], {}, model_name,
                                            category, sub_category)
                    new_graph.source = "TF"

                    return new_graph

                if operation.node_def.op == "Const":
                    continue

                # Converting operation to nodes
                new_node = self._OP_TO_NODE.convert(operation,
                                                    operation.node_def)
                node_index = len(nodes)
                nodes.append(new_node)

                # Add input_operation_names to start_node_indices
                if operation.name in input_operation_names:
                    start_node_indices.append(node_index)

                # Input node, also the start node to the graph
                if operation.node_def.op == "Placeholder":
                    new_node.label = "Input_Placeholder"
                    start_node_indices.append(node_index)

                # populating from_nodes and to_nodes
                for in_tensor in list(operation.inputs):
                    if in_tensor not in tensor_to_index:
                        tensor_to_index[in_tensor] = len(edges)
                        new_edge = self._TENSOR_TO_EDGE.convert(in_tensor)
                        edges.append(new_edge)

                    edge_index = tensor_to_index[in_tensor]
                    if edge_index not in to_nodes:
                        to_nodes.update({edge_index: []})

                    to_nodes[edge_index].append(node_index)

                for out_tensor in list(operation.outputs):
                    if out_tensor not in tensor_to_index:
                        tensor_to_index[out_tensor] = len(edges)
                        new_edge = self._TENSOR_TO_EDGE.convert(out_tensor)
                        edges.append(new_edge)

                    edge_index = tensor_to_index[out_tensor]
                    if edge_index not in from_nodes:
                        from_nodes.update({edge_index: []})

                    from_nodes[edge_index].append(node_index)

            # Creating and adjacency list using from_nodes and to_nodes
            adj_list = dict()
            for edge_index in range(len(edges)):
                if edge_index not in from_nodes or edge_index not in to_nodes:
                    continue

                for node1_index in from_nodes[edge_index]:
                    for node2_index in to_nodes[edge_index]:
                        if node1_index not in adj_list:
                            adj_list.update({node1_index: list()})

                        adj_list[node1_index].append([edge_index, node2_index])

            if len(start_node_indices) == 0:
                print(
                    "Graph contains no input placeholders, cannot parse graph."
                )
                return None

            graph = Graph.Graph(nodes, start_node_indices, edges, adj_list,
                                model_name, category, sub_category)

            # Removing nodes which are not reachable from input
            graph.process_nodes()
            graph.source = "TF"

            return graph
Пример #25
0
def longest_increasing_subsequence_path(coll):
    graph = Graph(min(coll))
    for i, a in enumerate(coll):
        for b in coll[i:]:
            if a < b: graph.add_edge(a, b, 1)
    return longest_paths(graph)