def create_graph(branches, drops_per_branch): graph = [] completed_uids = [] final_apps = [] for branch in range(branches): for i in range(drops_per_branch): data_uid = 'data_%d_branch_%d' % (i, branch) app_uid = 'app_%d_branch_%d' % (i, branch) data_drop = memory_drop(data_uid) app_drop = drop.dropdict({ 'node': hostname, 'oid': app_uid, 'uid': app_uid, 'type': 'app', 'app': 'dlg.apps.simple.SleepAndCopyApp', 'sleepTime': 0 }) data_drop.addConsumer(app_drop) graph.append(data_drop) graph.append(app_drop) if i == 0: completed_uids.append(data_uid) prev_app = data_drop elif i == drops_per_branch - 1: final_apps.append(app_drop) else: data_drop.addProducer(prev_app) final_drop = memory_drop('final') for final_app in final_apps: final_drop.addProducer(final_app) graph.append(final_drop) return graph, completed_uids
def fileSpec(uid, **kwargs): dropSpec = dropdict({ 'oid': str(uid), 'type': 'plain', 'storage': Categories.FILE }) dropSpec.update(kwargs) return dropSpec
def scpSpec(uid, **kwargs): dropSpec = dropdict({ 'oid': str(uid), 'type': 'app', 'app': 'dlg.apps.scp.ScpApp' }) dropSpec.update(kwargs) return dropSpec
def memory_drop(uid): return drop.dropdict({ 'node': hostname, 'oid': uid, 'uid': uid, 'type': 'plain', 'storage': 'memory' })
def memorySpec(uid, **kwargs): dropSpec = dropdict({ 'oid': str(uid), 'type': 'plain', 'storage': 'memory' }) dropSpec.update(kwargs) return dropSpec
def splitSpec(uid, **kwargs): dropSpec = dropdict({ 'oid': str(uid), 'type': 'app', 'app': 'test.integrate.chiles.chilesdo.Split' }) dropSpec.update(kwargs) return dropSpec
def directorySpec(uid, **kwargs): dropSpec = dropdict({ 'oid': str(uid), 'type': 'container', 'container': 'dlg.drop.DirectoryContainer' }) dropSpec.update(kwargs) return dropSpec
def cleanSpec(uid, **kwargs): dropSpec = dropdict({ 'oid': str(uid), 'type': 'app', 'app': 'dlg_integrate.wsclean.wscleandrop.WSClean', 'tw': 5 }) dropSpec.update(kwargs) return dropSpec
def fileDropSpec(uid, **kwargs): dropSpec = dropdict({ 'oid': str(uid), 'type': 'plain', 'storage': Categories.FILE, 'node': 'localhost', 'island': 'localhost' }) dropSpec.update(kwargs) return dropSpec
def fluxSpec(uid, **kwargs): dropSpec = dropdict( { "oid": str(uid), "type": "app", "app": "test.integrate.chiles.chilesdo.SourceFlux", } ) dropSpec.update(kwargs) return dropSpec
def fileDropSpec(uid, **kwargs): dropSpec = dropdict({ "oid": str(uid), "type": "plain", "storage": Categories.FILE, "node": "localhost", "island": "localhost", }) dropSpec.update(kwargs) return dropSpec
def directorySpec(uid, **kwargs): dropSpec = dropdict({ "oid": str(uid), "type": "container", "container": "dlg.drop.DirectoryContainer", "node": "localhost", "island": "localhost", }) dropSpec.update(kwargs) return dropSpec
def casapyDockerAppSpec(uid, script): cmd = ("cd; " + os.path.join(CASAPY, "casapy") + ' --colors=NoColor --nologger --nogui -c "%s"' % (script)) return dropdict({ "oid": str(uid), "type": "app", "app": "dlg.apps.dockerapp.DockerApp", "image": "dfms/casapy_centos7_dfms:0.1", "command": cmd, "user": "******", "node": "localhost", "island": "localhost", })
def create_memory_drop(self, node_id, oid='memory_drop'): oid_text = self.get_oid(oid) # uid_text = self.get_uuid() drop = dropdict({ "type": 'plain', "storage": 'memory', "oid": oid_text, # "uid": uid_text, "precious": False, "node": node_id, }) self.add_drop(drop) return drop
def casapyDockerAppSpec(uid, script): cmd = 'cd; ' + os.path.join( CASAPY, 'casapy') + ' --colors=NoColor --nologger --nogui -c "%s"' % (script) return dropdict({ 'oid': str(uid), 'type': 'app', 'app': 'dlg.apps.dockerapp.DockerApp', 'image': 'dfms/casapy_centos7_dfms:0.1', 'command': cmd, 'user': '******', 'node': 'localhost', 'island': 'localhost' })
def create_barrier_app(self, node_id, oid='barrier_app', input_error_threshold=100): oid_text = self.get_oid(oid) # uid_text = self.get_uuid() drop = dropdict({ "type": 'app', "app": get_module_name(BarrierAppDROP), "oid": oid_text, # "uid": uid_text, "input_error_threshold": input_error_threshold, "node": node_id, }) self.add_drop(drop) return drop
def create_bash_shell_app(self, node_id, command, oid='bash_shell_app', input_error_threshold=100): oid_text = self.get_oid(oid) # uid_text = self.get_uuid() drop = dropdict({ "type": 'app', "app": get_module_name(BashShellApp), "oid": oid_text, # "uid": uid_text, "command": command, "input_error_threshold": input_error_threshold, "node": node_id, }) self.add_drop(drop) return drop
def create_app(self, node_id, app, oid, input_error_threshold=100, **key_word_arguments): oid_text = self.get_oid(oid) # uid_text = self.get_uuid() drop = dropdict({ "type": 'app', "app": app, "oid": oid_text, # "uid": uid_text, "input_error_threshold": input_error_threshold, "node": node_id, }) drop.update(key_word_arguments) self.add_drop(drop) return drop
def test_get_roots(self): """ Check that the get_roots method from the droputils module works as intended """ """ A --> B """ pg_spec = [{ "oid": "A", "type": "plain", "storage": "memory", "consumers": ["B"] }, { "oid": "B", "type": "app", "app": "test.test_graph_loader.DummyApp" }] roots = droputils.get_roots(pg_spec) self.assertEqual(1, len(roots)) self.assertEqual('A', next(iter(roots))) """ A --> B The same, but now B references A """ pg_spec = [{ "oid": "A", "type": "plain", "storage": "memory" }, { "oid": "B", "type": "app", "app": "test.test_graph_loader.DummyApp", "inputs": ["A"] }] roots = droputils.get_roots(pg_spec) self.assertEqual(1, len(roots)) self.assertEqual('A', next(iter(roots))) """ A --> C --> D --| |--> E --> F B --------------| """ pg_spec = [{ "oid": "A", "type": "plain", "storage": "memory" }, { "oid": "B", "type": "plain", "storage": "memory" }, { "oid": "C", "type": "app", "app": "dlg.apps.crc.CRCApp", "inputs": ['A'] }, { "oid": "D", "type": "plain", "storage": "memory", "producers": ["C"] }, { "oid": "E", "type": "app", "app": "test.test_drop.SumupContainerChecksum", "inputs": ["D"] }, { "oid": "F", "type": "plain", "storage": "memory", "producers": ["E"] }] roots = droputils.get_roots(pg_spec) self.assertEqual(2, len(roots)) self.assertListEqual(['A', 'B'], sorted(roots)) # The same as before but using dropdicts pg_spec_dropdicts = [dropdict(dropspec) for dropspec in pg_spec] roots = droputils.get_roots(pg_spec_dropdicts) self.assertEqual(2, len(roots)) self.assertListEqual(['A', 'B'], sorted(roots))
def scpSpec(uid, **kwargs): dropSpec = dropdict({"oid": str(uid), "type": "app", "app": "dlg.apps.scp.ScpApp"}) dropSpec.update(kwargs) return dropSpec
def fileSpec(uid, **kwargs): dropSpec = dropdict({"oid": str(uid), "type": "plain", "storage": Categories.FILE}) dropSpec.update(kwargs) return dropSpec
def splitSpec(uid, **kwargs): dropSpec = dropdict( {"oid": str(uid), "type": "app", "app": "test.integrate.chiles.chilesdo.Split"} ) dropSpec.update(kwargs) return dropSpec
def sleepAndCopy(uid, **kwargs): dropSpec = dropdict({'oid':uid, 'type':'app', 'app':'dlg.apps.simple.SleepAndCopyApp'}) dropSpec.update(kwargs) return dropSpec