def create_graph(branches, drops_per_branch):
graph = []
completed_uids = []
final_apps = []
for branch in range(branches):
for i in range(drops_per_branch):
data_uid = 'data_%d_branch_%d' % (i, branch)
app_uid = 'app_%d_branch_%d' % (i, branch)
data_drop = memory_drop(data_uid)
app_drop = drop.dropdict({
'node': hostname,
'oid': app_uid,
'uid': app_uid,
'type': 'app',
'app': 'dlg.apps.simple.SleepAndCopyApp',
'sleepTime': 0
})
data_drop.addConsumer(app_drop)
graph.append(data_drop)
graph.append(app_drop)
if i == 0:
completed_uids.append(data_uid)
prev_app = data_drop
elif i == drops_per_branch - 1:
final_apps.append(app_drop)
else:
data_drop.addProducer(prev_app)
final_drop = memory_drop('final')
for final_app in final_apps:
final_drop.addProducer(final_app)
graph.append(final_drop)
return graph, completed_uids
def fileSpec(uid, **kwargs):
dropSpec = dropdict({
'oid': str(uid),
'type': 'plain',
'storage': Categories.FILE
})
dropSpec.update(kwargs)
return dropSpec
def scpSpec(uid, **kwargs):
dropSpec = dropdict({
'oid': str(uid),
'type': 'app',
'app': 'dlg.apps.scp.ScpApp'
})
dropSpec.update(kwargs)
return dropSpec
def memory_drop(uid):
return drop.dropdict({
'node': hostname,
'oid': uid,
'uid': uid,
'type': 'plain',
'storage': 'memory'
})
def memorySpec(uid, **kwargs):
dropSpec = dropdict({
'oid': str(uid),
'type': 'plain',
'storage': 'memory'
})
dropSpec.update(kwargs)
return dropSpec
def splitSpec(uid, **kwargs):
dropSpec = dropdict({
'oid': str(uid),
'type': 'app',
'app': 'test.integrate.chiles.chilesdo.Split'
})
dropSpec.update(kwargs)
return dropSpec
def directorySpec(uid, **kwargs):
dropSpec = dropdict({
'oid': str(uid),
'type': 'container',
'container': 'dlg.drop.DirectoryContainer'
})
dropSpec.update(kwargs)
return dropSpec
def cleanSpec(uid, **kwargs):
dropSpec = dropdict({
'oid': str(uid),
'type': 'app',
'app': 'dlg_integrate.wsclean.wscleandrop.WSClean',
'tw': 5
})
dropSpec.update(kwargs)
return dropSpec
def fileDropSpec(uid, **kwargs):
dropSpec = dropdict({
'oid': str(uid),
'type': 'plain',
'storage': Categories.FILE,
'node': 'localhost',
'island': 'localhost'
})
dropSpec.update(kwargs)
return dropSpec
def fluxSpec(uid, **kwargs):
dropSpec = dropdict(
{
"oid": str(uid),
"type": "app",
"app": "test.integrate.chiles.chilesdo.SourceFlux",
}
)
dropSpec.update(kwargs)
return dropSpec
def fileDropSpec(uid, **kwargs):
dropSpec = dropdict({
"oid": str(uid),
"type": "plain",
"storage": Categories.FILE,
"node": "localhost",
"island": "localhost",
})
dropSpec.update(kwargs)
return dropSpec
def directorySpec(uid, **kwargs):
dropSpec = dropdict({
"oid": str(uid),
"type": "container",
"container": "dlg.drop.DirectoryContainer",
"node": "localhost",
"island": "localhost",
})
dropSpec.update(kwargs)
return dropSpec
def casapyDockerAppSpec(uid, script):
cmd = ("cd; " + os.path.join(CASAPY, "casapy") +
' --colors=NoColor --nologger --nogui -c "%s"' % (script))
return dropdict({
"oid": str(uid),
"type": "app",
"app": "dlg.apps.dockerapp.DockerApp",
"image": "dfms/casapy_centos7_dfms:0.1",
"command": cmd,
"user": "******",
"node": "localhost",
"island": "localhost",
})
def create_memory_drop(self, node_id, oid='memory_drop'):
oid_text = self.get_oid(oid)
# uid_text = self.get_uuid()
drop = dropdict({
"type": 'plain',
"storage": 'memory',
"oid": oid_text,
# "uid": uid_text,
"precious": False,
"node": node_id,
})
self.add_drop(drop)
return drop
def casapyDockerAppSpec(uid, script):
cmd = 'cd; ' + os.path.join(
CASAPY,
'casapy') + ' --colors=NoColor --nologger --nogui -c "%s"' % (script)
return dropdict({
'oid': str(uid),
'type': 'app',
'app': 'dlg.apps.dockerapp.DockerApp',
'image': 'dfms/casapy_centos7_dfms:0.1',
'command': cmd,
'user': '******',
'node': 'localhost',
'island': 'localhost'
})
def create_barrier_app(self,
node_id,
oid='barrier_app',
input_error_threshold=100):
oid_text = self.get_oid(oid)
# uid_text = self.get_uuid()
drop = dropdict({
"type": 'app',
"app": get_module_name(BarrierAppDROP),
"oid": oid_text,
# "uid": uid_text,
"input_error_threshold": input_error_threshold,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_bash_shell_app(self,
node_id,
command,
oid='bash_shell_app',
input_error_threshold=100):
oid_text = self.get_oid(oid)
# uid_text = self.get_uuid()
drop = dropdict({
"type": 'app',
"app": get_module_name(BashShellApp),
"oid": oid_text,
# "uid": uid_text,
"command": command,
"input_error_threshold": input_error_threshold,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_app(self,
node_id,
app,
oid,
input_error_threshold=100,
**key_word_arguments):
oid_text = self.get_oid(oid)
# uid_text = self.get_uuid()
drop = dropdict({
"type": 'app',
"app": app,
"oid": oid_text,
# "uid": uid_text,
"input_error_threshold": input_error_threshold,
"node": node_id,
})
drop.update(key_word_arguments)
self.add_drop(drop)
return drop
def test_get_roots(self):
"""
Check that the get_roots method from the droputils module works as intended
"""
"""
A --> B
"""
pg_spec = [{
"oid": "A",
"type": "plain",
"storage": "memory",
"consumers": ["B"]
}, {
"oid": "B",
"type": "app",
"app": "test.test_graph_loader.DummyApp"
}]
roots = droputils.get_roots(pg_spec)
self.assertEqual(1, len(roots))
self.assertEqual('A', next(iter(roots)))
"""
A --> B
The same, but now B references A
"""
pg_spec = [{
"oid": "A",
"type": "plain",
"storage": "memory"
}, {
"oid": "B",
"type": "app",
"app": "test.test_graph_loader.DummyApp",
"inputs": ["A"]
}]
roots = droputils.get_roots(pg_spec)
self.assertEqual(1, len(roots))
self.assertEqual('A', next(iter(roots)))
"""
A --> C --> D --|
|--> E --> F
B --------------|
"""
pg_spec = [{
"oid": "A",
"type": "plain",
"storage": "memory"
}, {
"oid": "B",
"type": "plain",
"storage": "memory"
}, {
"oid": "C",
"type": "app",
"app": "dlg.apps.crc.CRCApp",
"inputs": ['A']
}, {
"oid": "D",
"type": "plain",
"storage": "memory",
"producers": ["C"]
}, {
"oid": "E",
"type": "app",
"app": "test.test_drop.SumupContainerChecksum",
"inputs": ["D"]
}, {
"oid": "F",
"type": "plain",
"storage": "memory",
"producers": ["E"]
}]
roots = droputils.get_roots(pg_spec)
self.assertEqual(2, len(roots))
self.assertListEqual(['A', 'B'], sorted(roots))
# The same as before but using dropdicts
pg_spec_dropdicts = [dropdict(dropspec) for dropspec in pg_spec]
roots = droputils.get_roots(pg_spec_dropdicts)
self.assertEqual(2, len(roots))
self.assertListEqual(['A', 'B'], sorted(roots))
def scpSpec(uid, **kwargs):
dropSpec = dropdict({"oid": str(uid), "type": "app", "app": "dlg.apps.scp.ScpApp"})
dropSpec.update(kwargs)
return dropSpec
def fileSpec(uid, **kwargs):
dropSpec = dropdict({"oid": str(uid), "type": "plain", "storage": Categories.FILE})
dropSpec.update(kwargs)
return dropSpec
def splitSpec(uid, **kwargs):
dropSpec = dropdict(
{"oid": str(uid), "type": "app", "app": "test.integrate.chiles.chilesdo.Split"}
)
dropSpec.update(kwargs)
return dropSpec
def sleepAndCopy(uid, **kwargs):
dropSpec = dropdict({'oid':uid, 'type':'app', 'app':'dlg.apps.simple.SleepAndCopyApp'})
dropSpec.update(kwargs)
return dropSpec
