Ejemplo n.º 1
0
def create_graph(branches, drops_per_branch):
    graph = []
    completed_uids = []
    final_apps = []
    for branch in range(branches):
        for i in range(drops_per_branch):
            data_uid = 'data_%d_branch_%d' % (i, branch)
            app_uid = 'app_%d_branch_%d' % (i, branch)
            data_drop = memory_drop(data_uid)
            app_drop = dropdict({
                'node': hostname,
                'oid': app_uid,
                'uid': app_uid,
                'type': 'app',
                'app': 'dlg.apps.simple.SleepAndCopyApp',
                'sleepTime': 0
            })
            data_drop.addConsumer(app_drop)
            graph.append(data_drop)
            graph.append(app_drop)
            if i == 0:
                completed_uids.append(data_uid)
                prev_app = data_drop
            elif i == drops_per_branch - 1:
                final_apps.append(app_drop)
            else:
                data_drop.addProducer(prev_app)

    final_drop = memory_drop('final')
    for final_app in final_apps:
        final_drop.addProducer(final_app)

    graph.append(final_drop)
    return graph, completed_uids
Ejemplo n.º 2
0
def sleepAndCopy(uid, **kwargs):
    dropSpec = dropdict({
        "oid": uid,
        "type": "app",
        "app": "dlg.apps.simple.SleepAndCopyApp"
    })
    dropSpec.update(kwargs)
    return dropSpec
Ejemplo n.º 3
0
def memory(uid, **kwargs):
    dropSpec = dropdict({
        "oid": uid,
        "type": "plain",
        "storage": Categories.MEMORY
    })
    dropSpec.update(kwargs)
    return dropSpec
Ejemplo n.º 4
0
def memory_drop(uid):
    return dropdict({
        'node': hostname,
        'oid': uid,
        'uid': uid,
        'type': 'plain',
        'storage': Categories.MEMORY
    })
Ejemplo n.º 5
0
def sleepAndCopy(uid, **kwargs):
    dropSpec = dropdict({
        "oid": uid,
        "type": "app",
        "app": "dlg.apps.simple.SleepAndCopyApp",
        "reprodata": default_repro.copy()
    })
    dropSpec.update(kwargs)
    return dropSpec
Ejemplo n.º 6
0
def memory(uid, **kwargs):
    dropSpec = dropdict({
        "oid": uid,
        "type": "plain",
        "storage": Categories.MEMORY,
        "reprodata": default_repro.copy()
    })
    dropSpec.update(kwargs)
    return dropSpec
Ejemplo n.º 7
0
def memory_drop(uid):
    return dropdict(
        {
            "node": hostname,
            "oid": uid,
            "uid": uid,
            "type": "plain",
            "storage": Categories.MEMORY,
        }
    )
Ejemplo n.º 8
0
def create_graph(branches, drops_per_branch):
    graph = []
    completed_uids = []
    final_apps = []
    for branch in range(branches):
        for i in range(drops_per_branch):
            data_uid = "data_%d_branch_%d" % (i, branch)
            app_uid = "app_%d_branch_%d" % (i, branch)
            data_drop = memory_drop(data_uid)
            app_drop = dropdict(
                {
                    "node": hostname,
                    "oid": app_uid,
                    "uid": app_uid,
                    "type": "app",
                    "app": "dlg.apps.simple.SleepAndCopyApp",
                    "sleepTime": 0,
                }
            )
            data_drop.addConsumer(app_drop)
            graph.append(data_drop)
            graph.append(app_drop)
            if i == 0:
                completed_uids.append(data_uid)
                prev_app = data_drop
            elif i == drops_per_branch - 1:
                final_apps.append(app_drop)
            else:
                data_drop.addProducer(prev_app)

    final_drop = memory_drop("final")
    for final_app in final_apps:
        final_drop.addProducer(final_app)

    graph.append(final_drop)
    add_test_reprodata(graph)
    return graph, completed_uids
Ejemplo n.º 9
0
    def test_get_roots(self):
        """
        Check that the get_roots method from the droputils module works as intended
        """

        """
        A --> B
        """
        pg_spec = [
            {
                "oid": "A",
                "type": "plain",
                "storage": Categories.MEMORY,
                "consumers": ["B"],
            },
            {"oid": "B", "type": "app", "app": "test.test_graph_loader.DummyApp"},
        ]
        roots = droputils.get_roots(pg_spec)
        self.assertEqual(1, len(roots))
        self.assertEqual("A", next(iter(roots)))

        """
        A --> B
        The same, but now B references A
        """
        pg_spec = [
            {"oid": "A", "type": "plain", "storage": Categories.MEMORY},
            {
                "oid": "B",
                "type": "app",
                "app": "test.test_graph_loader.DummyApp",
                "inputs": ["A"],
            },
        ]
        roots = droputils.get_roots(pg_spec)
        self.assertEqual(1, len(roots))
        self.assertEqual("A", next(iter(roots)))

        """
        A --> C --> D --|
                        |--> E --> F
        B --------------|
        """
        pg_spec = [
            {"oid": "A", "type": "plain", "storage": Categories.MEMORY},
            {"oid": "B", "type": "plain", "storage": Categories.MEMORY},
            {"oid": "C", "type": "app", "app": "dlg.apps.crc.CRCApp", "inputs": ["A"]},
            {
                "oid": "D",
                "type": "plain",
                "storage": Categories.MEMORY,
                "producers": ["C"],
            },
            {
                "oid": "E",
                "type": "app",
                "app": "test.test_drop.SumupContainerChecksum",
                "inputs": ["D"],
            },
            {
                "oid": "F",
                "type": "plain",
                "storage": Categories.MEMORY,
                "producers": ["E"],
            },
        ]
        roots = droputils.get_roots(pg_spec)
        self.assertEqual(2, len(roots))
        self.assertListEqual(["A", "B"], sorted(roots))

        # The same as before but using dropdicts
        pg_spec_dropdicts = [dropdict(dropspec) for dropspec in pg_spec]
        roots = droputils.get_roots(pg_spec_dropdicts)
        self.assertEqual(2, len(roots))
        self.assertListEqual(["A", "B"], sorted(roots))
Ejemplo n.º 10
0
Archivo: pgt.py Proyecto: ICRAR/daliuge
    def to_gojs_json(self, string_rep=True, outdict=None, visual=False):
        """
        Convert PGT (without any partitions) to JSON for visualisation in GOJS

        Sub-class PGTPs will override this function, and replace this with
        actual partitioning, and the visulisation becomes an option
        """
        G = self.dag
        ret = dict()
        ret["class"] = "go.GraphLinksModel"
        nodes = []
        links = []
        key_dict = dict()  # key - oid, value - GOJS key

        for i, drop in enumerate(self._drop_list):
            oid = drop["oid"]
            node = dict()
            node["key"] = i + 1
            key_dict[oid] = i + 1
            node["oid"] = oid
            tt = drop["type"]
            if DropType.PLAIN == tt:
                node["category"] = Categories.DATA
            elif DropType.APP == tt:
                node["category"] = Categories.COMPONENT
            node["text"] = drop["nm"]
            nodes.append(node)

        if self._extra_drops is None:
            extra_drops = []
            remove_edges = []
            add_edges = []  # a list of tuples
            add_nodes = []
            for drop in self._drop_list:
                oid = drop["oid"]
                myk = key_dict[oid]
                for i, oup in enumerate(G.successors(myk)):
                    link = dict()
                    link["from"] = myk
                    from_dt = 0 if drop["type"] == DropType.PLAIN else 1
                    to_dt = G.nodes[oup]["dt"]
                    if from_dt == to_dt:
                        to_drop = G.nodes[oup]["drop_spec"]
                        if from_dt == 0:
                            # add an extra app DROP
                            extra_oid = "{0}_TransApp_{1}".format(oid, i)
                            dropSpec = dropdict({
                                "oid": extra_oid,
                                "type": DropType.APP,
                                "app": "dlg.drop.BarrierAppDROP",
                                "nm": "go_app",
                                "tw": 1,
                            })
                            # create links
                            drop.addConsumer(dropSpec)
                            dropSpec.addInput(drop)
                            dropSpec.addOutput(to_drop)
                            to_drop.addProducer(dropSpec)
                            mydt = 1
                        else:
                            # add an extra data DROP
                            extra_oid = "{0}_TransData_{1}".format(oid, i)
                            dropSpec = dropdict({
                                "oid": extra_oid,
                                "type": DropType.PLAIN,
                                "storage": Categories.MEMORY,
                                "nm": "go_data",
                                "dw": 1,
                            })
                            drop.addOutput(dropSpec)
                            dropSpec.addProducer(drop)
                            dropSpec.addConsumer(to_drop)
                            to_drop.addInput(dropSpec)
                            mydt = 0
                        extra_drops.append(dropSpec)
                        lid = len(extra_drops) * -1
                        link["to"] = lid
                        endlink = dict()
                        endlink["from"] = lid
                        endlink["to"] = oup
                        links.append(endlink)
                        # global graph updates
                        # the new drop must have the same gid as the to_drop
                        add_nodes.append((lid, 1, mydt, dropSpec,
                                          G.nodes[oup].get("gid", None)))
                        remove_edges.append((myk, oup))
                        add_edges.append((myk, lid))
                        add_edges.append((lid, oup))
                    else:
                        link["to"] = oup
                    links.append(link)
            for gn in add_nodes:
                # logger.debug("added gid = {0} for new node {1}".format(gn[4], gn[0]))
                G.add_node(gn[0],
                           weight=gn[1],
                           dt=gn[2],
                           drop_spec=gn[3],
                           gid=gn[4])
            G.remove_edges_from(remove_edges)
            G.add_edges_from(add_edges)
            self._extra_drops = extra_drops
        else:
            for drop in self._drop_list:
                oid = drop["oid"]
                myk = key_dict[oid]
                for oup in G.successors(myk):
                    link = dict()
                    link["from"] = myk
                    link["to"] = oup
                    links.append(link)

        # going through the extra_drops
        for i, drop in enumerate(self._extra_drops):
            oid = drop["oid"]
            node = dict()
            node["key"] = (i + 1) * -1
            node["oid"] = oid
            tt = drop["type"]
            if DropType.PLAIN == tt:
                node["category"] = Categories.DATA
            elif DropType.APP == tt:
                node["category"] = Categories.COMPONENT
            node["text"] = drop["nm"]
            nodes.append(node)

        ret["nodeDataArray"] = nodes
        ret["linkDataArray"] = links
        self._gojs_json_obj = ret
        if string_rep:
            return json.dumps(ret, indent=2)
        else:
            return ret