def create_graph(branches, drops_per_branch):
graph = []
completed_uids = []
final_apps = []
for branch in range(branches):
for i in range(drops_per_branch):
data_uid = 'data_%d_branch_%d' % (i, branch)
app_uid = 'app_%d_branch_%d' % (i, branch)
data_drop = memory_drop(data_uid)
app_drop = drop.dropdict({'node':hostname, 'oid':app_uid, 'uid':app_uid, 'type':'app', 'app':'test.graphsRepository.SleepAndCopyApp', 'sleepTime':0})
data_drop.addConsumer(app_drop)
graph.append(data_drop)
graph.append(app_drop)
if i == 0:
completed_uids.append(data_uid)
prev_app = data_drop
elif i == drops_per_branch - 1:
final_apps.append(app_drop)
else:
data_drop.addProducer(prev_app)
final_drop = memory_drop('final')
for final_app in final_apps:
final_drop.addProducer(final_app)
graph.append(final_drop)
return graph, completed_uids
def create_graph(branches, drops_per_branch):
graph = []
completed_uids = []
final_apps = []
for branch in xrange(branches):
for i in xrange(drops_per_branch):
data_uid = 'data_%d_branch_%d' % (i, branch)
app_uid = 'app_%d_branch_%d' % (i, branch)
data_drop = memory_drop(data_uid)
app_drop = drop.dropdict({'node':hostname, 'oid':app_uid, 'uid':app_uid, 'type':'app', 'app':'test.graphsRepository.SleepAndCopyApp', 'sleepTime':0})
data_drop.addConsumer(app_drop)
graph.append(data_drop)
graph.append(app_drop)
if i == 0:
completed_uids.append(data_uid)
prev_app = data_drop
elif i == drops_per_branch - 1:
final_apps.append(app_drop)
else:
data_drop.addProducer(prev_app)
final_drop = memory_drop('final')
for final_app in final_apps:
final_drop.addProducer(final_app)
graph.append(final_drop)
return graph, completed_uids
def sleepAndCopy(uid, **kwargs):
dropSpec = dropdict({
'oid': uid,
'type': 'app',
'app': 'test.graphsRepository.SleepAndCopyApp'
})
dropSpec.update(kwargs)
return dropSpec
def memory_drop(uid):
return drop.dropdict({
'node': hostname,
'oid': uid,
'uid': uid,
'type': 'plain',
'storage': 'memory'
})
def directorySpec(uid, **kwargs):
dropSpec = dropdict({
'oid': str(uid),
'type': 'container',
'container': 'dfms.drop.DirectoryContainer'
})
dropSpec.update(kwargs)
return dropSpec
def cleanSpec(uid, **kwargs):
dropSpec = dropdict({
'oid': str(uid),
'type': 'app',
'app': 'test.integrate.chiles.chilesdo.Clean'
})
dropSpec.update(kwargs)
return dropSpec
def scpSpec(uid, **kwargs):
dropSpec = dropdict({
'oid': str(uid),
'type': 'app',
'app': 'dfms.apps.scp.ScpApp'
})
dropSpec.update(kwargs)
return dropSpec
def fluxSpec(uid, **kwargs):
dropSpec = dropdict({
'oid': str(uid),
'type': 'app',
'app': 'test.integrate.chiles.chilesdo.SourceFlux'
})
dropSpec.update(kwargs)
return dropSpec
def create_memory_drop(self, node_id, oid='memory_drop'):
drop = dropdict({
"type": 'plain',
"storage": 'memory',
"oid": self.get_oid(oid),
"uid": self.get_uuid(),
"node": node_id,
})
self.add_drop(drop)
return drop
def create_barrier_app(self, node_id, oid='barrier_app', input_error_threshold=100):
drop = dropdict({
"type": 'app',
"app": get_module_name(BarrierAppDROP),
"oid": self.get_oid(oid),
"uid": self.get_uuid(),
"input_error_threshold": input_error_threshold,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_app(self, node_id, app, oid, input_error_threshold=100, **key_word_arguments):
drop = dropdict({
"type": 'app',
"app": app,
"oid": self.get_oid(oid),
"uid": self.get_uuid(),
"input_error_threshold": input_error_threshold,
"node": node_id,
})
drop.update(key_word_arguments)
self.add_drop(drop)
return drop
def create_bash_shell_app(self, node_id, command, oid='bash_shell_app', input_error_threshold=100):
drop = dropdict({
"type": 'app',
"app": get_module_name(BashShellApp),
"oid": self.get_oid(oid),
"uid": self.get_uuid(),
"command": command,
"input_error_threshold": input_error_threshold,
"node": node_id,
})
self.add_drop(drop)
return drop
def build_graph(self):
start_drop = dropdict({
"type": 'plain',
"storage": 'memory',
"oid": get_oid('memory_in'),
"uid": get_uuid(),
})
self._start_oids.append(start_drop['uid'])
self.append(start_drop)
shutdown_drop = dropdict({
"type": 'app',
"app": get_module_name(BashShellApp),
"oid": get_oid('app_bash_shell_app'),
"uid": get_uuid(),
"command": 'sudo shutdown -h +5 "DFMS node shutting down" &',
"user": '******',
"input_error_threshold": 100,
})
shutdown_drop.addInput(start_drop)
self.append(shutdown_drop)
def build_graph(self):
start_drop = dropdict({
"type": 'plain',
"storage": 'memory',
"oid": get_oid('memory_in'),
"uid": get_uuid(),
})
self._start_oids.append(start_drop['uid'])
self.append(start_drop)
shutdown_drop = dropdict({
"type": 'app',
"app": get_module_name(BashShellApp),
"oid": get_oid('app_bash_shell_app'),
"uid": get_uuid(),
"command": 'sudo shutdown -h +5 "DFMS node shutting down" &',
"user": '******',
"input_error_threshold": 100,
})
shutdown_drop.addInput(start_drop)
self.append(shutdown_drop)
def create_memory_drop(self, node_id, oid='memory_drop'):
oid_text = self.get_oid(oid)
uid_text = self.get_uuid()
drop = dropdict({
"type": 'plain',
"storage": 'memory',
"oid": oid_text,
"uid": uid_text,
"precious": False,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_barrier_app(self, node_id, oid='barrier_app', input_error_threshold=100):
oid_text = self.get_oid(oid)
uid_text = self.get_uuid()
drop = dropdict({
"type": 'app',
"app": get_module_name(BarrierAppDROP),
"oid": oid_text,
"uid": uid_text,
"input_error_threshold": input_error_threshold,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_memory_drop(self, node_id, oid='memory_drop'):
oid_text = self.get_oid(oid)
uid_text = self.get_uuid()
drop = dropdict({
"type": 'plain',
"storage": 'memory',
"oid": oid_text,
"uid": uid_text,
"precious": False,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_bash_shell_app(self, node_id, command, oid='bash_shell_app', input_error_threshold=100):
oid_text = self.get_oid(oid)
uid_text = self.get_uuid()
drop = dropdict({
"type": 'app',
"app": get_module_name(BashShellApp),
"oid": oid_text,
"uid": uid_text,
"command": command,
"input_error_threshold": input_error_threshold,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_app(self, node_id, app, oid, input_error_threshold=100, **key_word_arguments):
oid_text = self.get_oid(oid)
uid_text = self.get_uuid()
drop = dropdict({
"type": 'app',
"app": app,
"oid": oid_text,
"uid": uid_text,
"input_error_threshold": input_error_threshold,
"node": node_id,
})
drop.update(key_word_arguments)
self.add_drop(drop)
return drop
def create_file_drop(self, node_id, filepath, oid='file'):
oid_text = self.get_oid(oid)
uid_text = self.get_uuid()
drop = dropdict({
"type": 'plain',
"storage": 'file',
"oid": oid_text,
"uid": uid_text,
"precious": False,
"filepath": filepath,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_file_drop(self, node_id, filepath, oid='file'):
oid_text = self.get_oid(oid)
uid_text = self.get_uuid()
drop = dropdict({
"type": 'plain',
"storage": 'file',
"oid": oid_text,
"uid": uid_text,
"precious": False,
"filepath": filepath,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_json_drop(self, node_id, oid='json'):
oid_text = self.get_oid(oid)
drop = dropdict({
"type": 'plain',
"storage": 'json',
"oid": oid_text,
"uid": self.get_uuid(),
"precious": False,
"dirname": os.path.join(self._volume, oid_text),
"check_exists": False,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_directory_container(self, node_id, oid='directory_container', expire_after_use=True):
oid_text = self.get_oid(oid)
drop = dropdict({
"type": 'container',
"container": get_module_name(DirectoryContainer),
"oid": oid_text,
"uid": self.get_uuid(),
"precious": False,
"dirname": os.path.join(self._volume, oid_text),
"check_exists": False,
"expireAfterUse": expire_after_use,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_s3_drop(self, node_id, bucket_name, key, profile_name, oid='s3'):
drop = dropdict({
"type": 'plain',
"storage": 's3',
"oid": self.get_oid(oid),
"uid": self.get_uuid(),
"expireAfterUse": True,
"precious": False,
"bucket": bucket_name,
"key": key,
"profile_name": profile_name,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_json_drop(self, node_id, oid='json'):
oid_text = self.get_oid(oid)
uid_text = self.get_uuid()
drop = dropdict({
"type": 'plain',
"storage": 'json',
"oid": oid_text,
"uid": uid_text,
"precious": False,
"dirname": os.path.join(self._volume, oid_text),
"check_exists": False,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_docker_app(self, node_id, app, oid, image, command, user='******', input_error_threshold=100, **key_word_arguments):
drop = dropdict({
"type": 'app',
"app": app,
"oid": self.get_oid(oid),
"uid": self.get_uuid(),
"image": image,
"command": command,
"user": user,
"input_error_threshold": input_error_threshold,
"node": node_id,
})
drop.update(key_word_arguments)
self.add_drop(drop)
return drop
def create_directory_container(self, node_id, oid='directory_container', expire_after_use=True):
oid_text = self.get_oid(oid)
uid_text = self.get_uuid()
drop = dropdict({
"type": 'container',
"container": get_module_name(DirectoryContainer),
"oid": oid_text,
"uid": uid_text,
"precious": False,
"dirname": os.path.join(self._volume, oid_text),
"check_exists": False,
"expireAfterUse": expire_after_use,
"node": node_id,
})
self.add_drop(drop)
return drop
def create_docker_app(self, node_id, app, oid, image, command, user='******', input_error_threshold=100, **key_word_arguments):
oid_text = self.get_oid(oid)
uid_text = self.get_uuid()
drop = dropdict({
"type": 'app',
"app": app,
"oid": oid_text,
"uid": uid_text,
"image": image,
"command": command,
"user": user,
"input_error_threshold": input_error_threshold,
"node": node_id,
})
drop.update(key_word_arguments)
self.add_drop(drop)
return drop
def create_s3_drop(self, node_id, bucket_name, key, profile_name, oid='s3'):
oid_text = self.get_oid(oid)
uid_text = self.get_uuid()
drop = dropdict({
"type": 'plain',
"storage": 's3',
"oid": oid_text,
"uid": uid_text,
"expireAfterUse": True,
"precious": False,
"bucket": bucket_name,
"key": key,
"profile_name": profile_name,
"node": node_id,
})
self.add_drop(drop)
return drop
def fluxSpec(uid, **kwargs):
dropSpec = dropdict({'oid':str(uid), 'type':'app', 'app':'test.integrate.chiles.chilesdo.SourceFlux'})
dropSpec.update(kwargs)
return dropSpec
def fileSpec(uid, **kwargs):
dropSpec = dropdict({'oid':str(uid), 'type':'plain', 'storage':'file'})
dropSpec.update(kwargs)
return dropSpec
def memory_drop(uid):
return drop.dropdict({'node':hostname, 'oid':uid, 'uid':uid, 'type':'plain', 'storage':'memory'})
def memory(uid, **kwargs):
dropSpec = dropdict({'oid': uid, 'type': 'plain', 'storage': 'memory'})
dropSpec.update(kwargs)
return dropSpec
def test_get_roots(self):
"""
Check that the get_roots method from the droputils module works as intended
"""
"""
A --> B
"""
pg_spec = [{
"oid": "A",
"type": "plain",
"storage": "memory",
"consumers": ["B"]
}, {
"oid": "B",
"type": "app",
"app": "test.test_graph_loader.DummyApp"
}]
roots = droputils.get_roots(pg_spec)
self.assertEqual(1, len(roots))
self.assertEqual('A', next(iter(roots)))
"""
A --> B
The same, but now B references A
"""
pg_spec = [{
"oid": "A",
"type": "plain",
"storage": "memory"
}, {
"oid": "B",
"type": "app",
"app": "test.test_graph_loader.DummyApp",
"inputs": ["A"]
}]
roots = droputils.get_roots(pg_spec)
self.assertEqual(1, len(roots))
self.assertEqual('A', next(iter(roots)))
"""
A --> C --> D --|
|--> E --> F
B --------------|
"""
pg_spec = [{
"oid": "A",
"type": "plain",
"storage": "memory"
}, {
"oid": "B",
"type": "plain",
"storage": "memory"
}, {
"oid": "C",
"type": "app",
"app": "dfms.apps.crc.CRCApp",
"inputs": ['A']
}, {
"oid": "D",
"type": "plain",
"storage": "memory",
"producers": ["C"]
}, {
"oid": "E",
"type": "app",
"app": "test.test_drop.SumupContainerChecksum",
"inputs": ["D"]
}, {
"oid": "F",
"type": "plain",
"storage": "memory",
"producers": ["E"]
}]
roots = droputils.get_roots(pg_spec)
self.assertEqual(2, len(roots))
self.assertListEqual(['A', 'B'], sorted(roots))
# The same as before but using dropdicts
pg_spec_dropdicts = [dropdict(dropspec) for dropspec in pg_spec]
roots = droputils.get_roots(pg_spec_dropdicts)
self.assertEqual(2, len(roots))
self.assertListEqual(['A', 'B'], sorted(roots))
def sleepAndCopy(uid, **kwargs):
dropSpec = dropdict({'oid':uid, 'type':'app', 'app':'test.graphsRepository.SleepAndCopyApp'})
dropSpec.update(kwargs)
return dropSpec
def scpSpec(uid, **kwargs):
dropSpec = dropdict({'oid':str(uid), 'type':'app', 'app':'dfms.apps.scp.ScpApp'})
dropSpec.update(kwargs)
return dropSpec
def directorySpec(uid, **kwargs):
dropSpec = dropdict({'oid':str(uid), 'type':'container', 'container':'dfms.drop.DirectoryContainer', 'node': 'localhost', 'island': 'localhost'})
dropSpec.update(kwargs)
return dropSpec
def cleanSpec(uid, **kwargs):
dropSpec = dropdict({'oid':str(uid), 'type':'app', 'app':'test.integrate.chiles.chilesdo.Clean'})
dropSpec.update(kwargs)
return dropSpec
def fileDropSpec(uid, **kwargs):
dropSpec = dropdict({'oid':str(uid), 'type':'plain', 'storage':'file', 'node': 'localhost', 'island': 'localhost'})
dropSpec.update(kwargs)
return dropSpec
def casapyDockerAppSpec(uid, script):
cmd = 'cd; ' + os.path.join(CASAPY, 'casapy') + ' --colors=NoColor --nologger --nogui -c "%s"' % (script)
return dropdict({'oid':str(uid), 'type':'app', 'app':'dfms.apps.dockerapp.DockerApp',
'image':'dfms/casapy_centos7_dfms:0.1',
'command':cmd, 'user': '******',
'node': 'localhost', 'island': 'localhost'})
def socket(uid, port=1111):
return dropdict({'oid':uid, 'type':'socket', 'port':port})
def fileDropSpec(uid, **kwargs):
dropSpec = dropdict({'oid':str(uid), 'type':'plain', 'storage':'file', 'node': 'localhost', 'island': 'localhost'})
dropSpec.update(kwargs)
return dropSpec
def memory(uid, **kwargs):
dropSpec = dropdict({'oid':uid, 'type':'plain', 'storage':'memory'})
dropSpec.update(kwargs)
return dropSpec
def casapyDockerAppSpec(uid, script):
cmd = 'cd; ' + os.path.join(CASAPY, 'casapy') + ' --colors=NoColor --nologger --nogui -c "%s"' % (script)
return dropdict({'oid':str(uid), 'type':'app', 'app':'dfms.apps.dockerapp.DockerApp',
'image':'dfms/casapy_centos7_dfms:0.1',
'command':cmd, 'user': '******',
'node': 'localhost', 'island': 'localhost'})
def fileSpec(uid, **kwargs):
dropSpec = dropdict({'oid': str(uid), 'type': 'plain', 'storage': 'file'})
dropSpec.update(kwargs)
return dropSpec
