def create_graph(branches, drops_per_branch):
graph = []
completed_uids = []
final_apps = []
for branch in range(branches):
for i in range(drops_per_branch):
data_uid = 'data_%d_branch_%d' % (i, branch)
app_uid = 'app_%d_branch_%d' % (i, branch)
data_drop = memory_drop(data_uid)
app_drop = dropdict({
'node': hostname,
'oid': app_uid,
'uid': app_uid,
'type': 'app',
'app': 'dlg.apps.simple.SleepAndCopyApp',
'sleepTime': 0
})
data_drop.addConsumer(app_drop)
graph.append(data_drop)
graph.append(app_drop)
if i == 0:
completed_uids.append(data_uid)
prev_app = data_drop
elif i == drops_per_branch - 1:
final_apps.append(app_drop)
else:
data_drop.addProducer(prev_app)
final_drop = memory_drop('final')
for final_app in final_apps:
final_drop.addProducer(final_app)
graph.append(final_drop)
return graph, completed_uids
def sleepAndCopy(uid, **kwargs):
dropSpec = dropdict({
"oid": uid,
"type": "app",
"app": "dlg.apps.simple.SleepAndCopyApp"
})
dropSpec.update(kwargs)
return dropSpec
def memory(uid, **kwargs):
dropSpec = dropdict({
"oid": uid,
"type": "plain",
"storage": Categories.MEMORY
})
dropSpec.update(kwargs)
return dropSpec
def memory_drop(uid):
return dropdict({
'node': hostname,
'oid': uid,
'uid': uid,
'type': 'plain',
'storage': Categories.MEMORY
})
def sleepAndCopy(uid, **kwargs):
dropSpec = dropdict({
"oid": uid,
"type": "app",
"app": "dlg.apps.simple.SleepAndCopyApp",
"reprodata": default_repro.copy()
})
dropSpec.update(kwargs)
return dropSpec
def memory(uid, **kwargs):
dropSpec = dropdict({
"oid": uid,
"type": "plain",
"storage": Categories.MEMORY,
"reprodata": default_repro.copy()
})
dropSpec.update(kwargs)
return dropSpec
def memory_drop(uid):
return dropdict(
{
"node": hostname,
"oid": uid,
"uid": uid,
"type": "plain",
"storage": Categories.MEMORY,
}
)
def create_graph(branches, drops_per_branch):
graph = []
completed_uids = []
final_apps = []
for branch in range(branches):
for i in range(drops_per_branch):
data_uid = "data_%d_branch_%d" % (i, branch)
app_uid = "app_%d_branch_%d" % (i, branch)
data_drop = memory_drop(data_uid)
app_drop = dropdict(
{
"node": hostname,
"oid": app_uid,
"uid": app_uid,
"type": "app",
"app": "dlg.apps.simple.SleepAndCopyApp",
"sleepTime": 0,
}
)
data_drop.addConsumer(app_drop)
graph.append(data_drop)
graph.append(app_drop)
if i == 0:
completed_uids.append(data_uid)
prev_app = data_drop
elif i == drops_per_branch - 1:
final_apps.append(app_drop)
else:
data_drop.addProducer(prev_app)
final_drop = memory_drop("final")
for final_app in final_apps:
final_drop.addProducer(final_app)
graph.append(final_drop)
add_test_reprodata(graph)
return graph, completed_uids
def test_get_roots(self):
"""
Check that the get_roots method from the droputils module works as intended
"""
"""
A --> B
"""
pg_spec = [
{
"oid": "A",
"type": "plain",
"storage": Categories.MEMORY,
"consumers": ["B"],
},
{"oid": "B", "type": "app", "app": "test.test_graph_loader.DummyApp"},
]
roots = droputils.get_roots(pg_spec)
self.assertEqual(1, len(roots))
self.assertEqual("A", next(iter(roots)))
"""
A --> B
The same, but now B references A
"""
pg_spec = [
{"oid": "A", "type": "plain", "storage": Categories.MEMORY},
{
"oid": "B",
"type": "app",
"app": "test.test_graph_loader.DummyApp",
"inputs": ["A"],
},
]
roots = droputils.get_roots(pg_spec)
self.assertEqual(1, len(roots))
self.assertEqual("A", next(iter(roots)))
"""
A --> C --> D --|
|--> E --> F
B --------------|
"""
pg_spec = [
{"oid": "A", "type": "plain", "storage": Categories.MEMORY},
{"oid": "B", "type": "plain", "storage": Categories.MEMORY},
{"oid": "C", "type": "app", "app": "dlg.apps.crc.CRCApp", "inputs": ["A"]},
{
"oid": "D",
"type": "plain",
"storage": Categories.MEMORY,
"producers": ["C"],
},
{
"oid": "E",
"type": "app",
"app": "test.test_drop.SumupContainerChecksum",
"inputs": ["D"],
},
{
"oid": "F",
"type": "plain",
"storage": Categories.MEMORY,
"producers": ["E"],
},
]
roots = droputils.get_roots(pg_spec)
self.assertEqual(2, len(roots))
self.assertListEqual(["A", "B"], sorted(roots))
# The same as before but using dropdicts
pg_spec_dropdicts = [dropdict(dropspec) for dropspec in pg_spec]
roots = droputils.get_roots(pg_spec_dropdicts)
self.assertEqual(2, len(roots))
self.assertListEqual(["A", "B"], sorted(roots))
def to_gojs_json(self, string_rep=True, outdict=None, visual=False):
"""
Convert PGT (without any partitions) to JSON for visualisation in GOJS
Sub-class PGTPs will override this function, and replace this with
actual partitioning, and the visulisation becomes an option
"""
G = self.dag
ret = dict()
ret["class"] = "go.GraphLinksModel"
nodes = []
links = []
key_dict = dict() # key - oid, value - GOJS key
for i, drop in enumerate(self._drop_list):
oid = drop["oid"]
node = dict()
node["key"] = i + 1
key_dict[oid] = i + 1
node["oid"] = oid
tt = drop["type"]
if DropType.PLAIN == tt:
node["category"] = Categories.DATA
elif DropType.APP == tt:
node["category"] = Categories.COMPONENT
node["text"] = drop["nm"]
nodes.append(node)
if self._extra_drops is None:
extra_drops = []
remove_edges = []
add_edges = [] # a list of tuples
add_nodes = []
for drop in self._drop_list:
oid = drop["oid"]
myk = key_dict[oid]
for i, oup in enumerate(G.successors(myk)):
link = dict()
link["from"] = myk
from_dt = 0 if drop["type"] == DropType.PLAIN else 1
to_dt = G.nodes[oup]["dt"]
if from_dt == to_dt:
to_drop = G.nodes[oup]["drop_spec"]
if from_dt == 0:
# add an extra app DROP
extra_oid = "{0}_TransApp_{1}".format(oid, i)
dropSpec = dropdict({
"oid": extra_oid,
"type": DropType.APP,
"app": "dlg.drop.BarrierAppDROP",
"nm": "go_app",
"tw": 1,
})
# create links
drop.addConsumer(dropSpec)
dropSpec.addInput(drop)
dropSpec.addOutput(to_drop)
to_drop.addProducer(dropSpec)
mydt = 1
else:
# add an extra data DROP
extra_oid = "{0}_TransData_{1}".format(oid, i)
dropSpec = dropdict({
"oid": extra_oid,
"type": DropType.PLAIN,
"storage": Categories.MEMORY,
"nm": "go_data",
"dw": 1,
})
drop.addOutput(dropSpec)
dropSpec.addProducer(drop)
dropSpec.addConsumer(to_drop)
to_drop.addInput(dropSpec)
mydt = 0
extra_drops.append(dropSpec)
lid = len(extra_drops) * -1
link["to"] = lid
endlink = dict()
endlink["from"] = lid
endlink["to"] = oup
links.append(endlink)
# global graph updates
# the new drop must have the same gid as the to_drop
add_nodes.append((lid, 1, mydt, dropSpec,
G.nodes[oup].get("gid", None)))
remove_edges.append((myk, oup))
add_edges.append((myk, lid))
add_edges.append((lid, oup))
else:
link["to"] = oup
links.append(link)
for gn in add_nodes:
# logger.debug("added gid = {0} for new node {1}".format(gn[4], gn[0]))
G.add_node(gn[0],
weight=gn[1],
dt=gn[2],
drop_spec=gn[3],
gid=gn[4])
G.remove_edges_from(remove_edges)
G.add_edges_from(add_edges)
self._extra_drops = extra_drops
else:
for drop in self._drop_list:
oid = drop["oid"]
myk = key_dict[oid]
for oup in G.successors(myk):
link = dict()
link["from"] = myk
link["to"] = oup
links.append(link)
# going through the extra_drops
for i, drop in enumerate(self._extra_drops):
oid = drop["oid"]
node = dict()
node["key"] = (i + 1) * -1
node["oid"] = oid
tt = drop["type"]
if DropType.PLAIN == tt:
node["category"] = Categories.DATA
elif DropType.APP == tt:
node["category"] = Categories.COMPONENT
node["text"] = drop["nm"]
nodes.append(node)
ret["nodeDataArray"] = nodes
ret["linkDataArray"] = links
self._gojs_json_obj = ret
if string_rep:
return json.dumps(ret, indent=2)
else:
return ret