def create_graph(branches, drops_per_branch): graph = [] completed_uids = [] final_apps = [] for branch in range(branches): for i in range(drops_per_branch): data_uid = 'data_%d_branch_%d' % (i, branch) app_uid = 'app_%d_branch_%d' % (i, branch) data_drop = memory_drop(data_uid) app_drop = drop.dropdict({'node':hostname, 'oid':app_uid, 'uid':app_uid, 'type':'app', 'app':'test.graphsRepository.SleepAndCopyApp', 'sleepTime':0}) data_drop.addConsumer(app_drop) graph.append(data_drop) graph.append(app_drop) if i == 0: completed_uids.append(data_uid) prev_app = data_drop elif i == drops_per_branch - 1: final_apps.append(app_drop) else: data_drop.addProducer(prev_app) final_drop = memory_drop('final') for final_app in final_apps: final_drop.addProducer(final_app) graph.append(final_drop) return graph, completed_uids
def create_graph(branches, drops_per_branch): graph = [] completed_uids = [] final_apps = [] for branch in xrange(branches): for i in xrange(drops_per_branch): data_uid = 'data_%d_branch_%d' % (i, branch) app_uid = 'app_%d_branch_%d' % (i, branch) data_drop = memory_drop(data_uid) app_drop = drop.dropdict({'node':hostname, 'oid':app_uid, 'uid':app_uid, 'type':'app', 'app':'test.graphsRepository.SleepAndCopyApp', 'sleepTime':0}) data_drop.addConsumer(app_drop) graph.append(data_drop) graph.append(app_drop) if i == 0: completed_uids.append(data_uid) prev_app = data_drop elif i == drops_per_branch - 1: final_apps.append(app_drop) else: data_drop.addProducer(prev_app) final_drop = memory_drop('final') for final_app in final_apps: final_drop.addProducer(final_app) graph.append(final_drop) return graph, completed_uids
def sleepAndCopy(uid, **kwargs): dropSpec = dropdict({ 'oid': uid, 'type': 'app', 'app': 'test.graphsRepository.SleepAndCopyApp' }) dropSpec.update(kwargs) return dropSpec
def memory_drop(uid): return drop.dropdict({ 'node': hostname, 'oid': uid, 'uid': uid, 'type': 'plain', 'storage': 'memory' })
def directorySpec(uid, **kwargs): dropSpec = dropdict({ 'oid': str(uid), 'type': 'container', 'container': 'dfms.drop.DirectoryContainer' }) dropSpec.update(kwargs) return dropSpec
def cleanSpec(uid, **kwargs): dropSpec = dropdict({ 'oid': str(uid), 'type': 'app', 'app': 'test.integrate.chiles.chilesdo.Clean' }) dropSpec.update(kwargs) return dropSpec
def scpSpec(uid, **kwargs): dropSpec = dropdict({ 'oid': str(uid), 'type': 'app', 'app': 'dfms.apps.scp.ScpApp' }) dropSpec.update(kwargs) return dropSpec
def fluxSpec(uid, **kwargs): dropSpec = dropdict({ 'oid': str(uid), 'type': 'app', 'app': 'test.integrate.chiles.chilesdo.SourceFlux' }) dropSpec.update(kwargs) return dropSpec
def create_memory_drop(self, node_id, oid='memory_drop'): drop = dropdict({ "type": 'plain', "storage": 'memory', "oid": self.get_oid(oid), "uid": self.get_uuid(), "node": node_id, }) self.add_drop(drop) return drop
def create_barrier_app(self, node_id, oid='barrier_app', input_error_threshold=100): drop = dropdict({ "type": 'app', "app": get_module_name(BarrierAppDROP), "oid": self.get_oid(oid), "uid": self.get_uuid(), "input_error_threshold": input_error_threshold, "node": node_id, }) self.add_drop(drop) return drop
def create_app(self, node_id, app, oid, input_error_threshold=100, **key_word_arguments): drop = dropdict({ "type": 'app', "app": app, "oid": self.get_oid(oid), "uid": self.get_uuid(), "input_error_threshold": input_error_threshold, "node": node_id, }) drop.update(key_word_arguments) self.add_drop(drop) return drop
def create_bash_shell_app(self, node_id, command, oid='bash_shell_app', input_error_threshold=100): drop = dropdict({ "type": 'app', "app": get_module_name(BashShellApp), "oid": self.get_oid(oid), "uid": self.get_uuid(), "command": command, "input_error_threshold": input_error_threshold, "node": node_id, }) self.add_drop(drop) return drop
def build_graph(self): start_drop = dropdict({ "type": 'plain', "storage": 'memory', "oid": get_oid('memory_in'), "uid": get_uuid(), }) self._start_oids.append(start_drop['uid']) self.append(start_drop) shutdown_drop = dropdict({ "type": 'app', "app": get_module_name(BashShellApp), "oid": get_oid('app_bash_shell_app'), "uid": get_uuid(), "command": 'sudo shutdown -h +5 "DFMS node shutting down" &', "user": '******', "input_error_threshold": 100, }) shutdown_drop.addInput(start_drop) self.append(shutdown_drop)
def create_memory_drop(self, node_id, oid='memory_drop'): oid_text = self.get_oid(oid) uid_text = self.get_uuid() drop = dropdict({ "type": 'plain', "storage": 'memory', "oid": oid_text, "uid": uid_text, "precious": False, "node": node_id, }) self.add_drop(drop) return drop
def create_barrier_app(self, node_id, oid='barrier_app', input_error_threshold=100): oid_text = self.get_oid(oid) uid_text = self.get_uuid() drop = dropdict({ "type": 'app', "app": get_module_name(BarrierAppDROP), "oid": oid_text, "uid": uid_text, "input_error_threshold": input_error_threshold, "node": node_id, }) self.add_drop(drop) return drop
def create_bash_shell_app(self, node_id, command, oid='bash_shell_app', input_error_threshold=100): oid_text = self.get_oid(oid) uid_text = self.get_uuid() drop = dropdict({ "type": 'app', "app": get_module_name(BashShellApp), "oid": oid_text, "uid": uid_text, "command": command, "input_error_threshold": input_error_threshold, "node": node_id, }) self.add_drop(drop) return drop
def create_app(self, node_id, app, oid, input_error_threshold=100, **key_word_arguments): oid_text = self.get_oid(oid) uid_text = self.get_uuid() drop = dropdict({ "type": 'app', "app": app, "oid": oid_text, "uid": uid_text, "input_error_threshold": input_error_threshold, "node": node_id, }) drop.update(key_word_arguments) self.add_drop(drop) return drop
def create_file_drop(self, node_id, filepath, oid='file'): oid_text = self.get_oid(oid) uid_text = self.get_uuid() drop = dropdict({ "type": 'plain', "storage": 'file', "oid": oid_text, "uid": uid_text, "precious": False, "filepath": filepath, "node": node_id, }) self.add_drop(drop) return drop
def create_json_drop(self, node_id, oid='json'): oid_text = self.get_oid(oid) drop = dropdict({ "type": 'plain', "storage": 'json', "oid": oid_text, "uid": self.get_uuid(), "precious": False, "dirname": os.path.join(self._volume, oid_text), "check_exists": False, "node": node_id, }) self.add_drop(drop) return drop
def create_directory_container(self, node_id, oid='directory_container', expire_after_use=True): oid_text = self.get_oid(oid) drop = dropdict({ "type": 'container', "container": get_module_name(DirectoryContainer), "oid": oid_text, "uid": self.get_uuid(), "precious": False, "dirname": os.path.join(self._volume, oid_text), "check_exists": False, "expireAfterUse": expire_after_use, "node": node_id, }) self.add_drop(drop) return drop
def create_s3_drop(self, node_id, bucket_name, key, profile_name, oid='s3'): drop = dropdict({ "type": 'plain', "storage": 's3', "oid": self.get_oid(oid), "uid": self.get_uuid(), "expireAfterUse": True, "precious": False, "bucket": bucket_name, "key": key, "profile_name": profile_name, "node": node_id, }) self.add_drop(drop) return drop
def create_json_drop(self, node_id, oid='json'): oid_text = self.get_oid(oid) uid_text = self.get_uuid() drop = dropdict({ "type": 'plain', "storage": 'json', "oid": oid_text, "uid": uid_text, "precious": False, "dirname": os.path.join(self._volume, oid_text), "check_exists": False, "node": node_id, }) self.add_drop(drop) return drop
def create_docker_app(self, node_id, app, oid, image, command, user='******', input_error_threshold=100, **key_word_arguments): drop = dropdict({ "type": 'app', "app": app, "oid": self.get_oid(oid), "uid": self.get_uuid(), "image": image, "command": command, "user": user, "input_error_threshold": input_error_threshold, "node": node_id, }) drop.update(key_word_arguments) self.add_drop(drop) return drop
def create_directory_container(self, node_id, oid='directory_container', expire_after_use=True): oid_text = self.get_oid(oid) uid_text = self.get_uuid() drop = dropdict({ "type": 'container', "container": get_module_name(DirectoryContainer), "oid": oid_text, "uid": uid_text, "precious": False, "dirname": os.path.join(self._volume, oid_text), "check_exists": False, "expireAfterUse": expire_after_use, "node": node_id, }) self.add_drop(drop) return drop
def create_docker_app(self, node_id, app, oid, image, command, user='******', input_error_threshold=100, **key_word_arguments): oid_text = self.get_oid(oid) uid_text = self.get_uuid() drop = dropdict({ "type": 'app', "app": app, "oid": oid_text, "uid": uid_text, "image": image, "command": command, "user": user, "input_error_threshold": input_error_threshold, "node": node_id, }) drop.update(key_word_arguments) self.add_drop(drop) return drop
def create_s3_drop(self, node_id, bucket_name, key, profile_name, oid='s3'): oid_text = self.get_oid(oid) uid_text = self.get_uuid() drop = dropdict({ "type": 'plain', "storage": 's3', "oid": oid_text, "uid": uid_text, "expireAfterUse": True, "precious": False, "bucket": bucket_name, "key": key, "profile_name": profile_name, "node": node_id, }) self.add_drop(drop) return drop
def fluxSpec(uid, **kwargs): dropSpec = dropdict({'oid':str(uid), 'type':'app', 'app':'test.integrate.chiles.chilesdo.SourceFlux'}) dropSpec.update(kwargs) return dropSpec
def fileSpec(uid, **kwargs): dropSpec = dropdict({'oid':str(uid), 'type':'plain', 'storage':'file'}) dropSpec.update(kwargs) return dropSpec
def memory_drop(uid): return drop.dropdict({'node':hostname, 'oid':uid, 'uid':uid, 'type':'plain', 'storage':'memory'})
def memory(uid, **kwargs): dropSpec = dropdict({'oid': uid, 'type': 'plain', 'storage': 'memory'}) dropSpec.update(kwargs) return dropSpec
def test_get_roots(self): """ Check that the get_roots method from the droputils module works as intended """ """ A --> B """ pg_spec = [{ "oid": "A", "type": "plain", "storage": "memory", "consumers": ["B"] }, { "oid": "B", "type": "app", "app": "test.test_graph_loader.DummyApp" }] roots = droputils.get_roots(pg_spec) self.assertEqual(1, len(roots)) self.assertEqual('A', next(iter(roots))) """ A --> B The same, but now B references A """ pg_spec = [{ "oid": "A", "type": "plain", "storage": "memory" }, { "oid": "B", "type": "app", "app": "test.test_graph_loader.DummyApp", "inputs": ["A"] }] roots = droputils.get_roots(pg_spec) self.assertEqual(1, len(roots)) self.assertEqual('A', next(iter(roots))) """ A --> C --> D --| |--> E --> F B --------------| """ pg_spec = [{ "oid": "A", "type": "plain", "storage": "memory" }, { "oid": "B", "type": "plain", "storage": "memory" }, { "oid": "C", "type": "app", "app": "dfms.apps.crc.CRCApp", "inputs": ['A'] }, { "oid": "D", "type": "plain", "storage": "memory", "producers": ["C"] }, { "oid": "E", "type": "app", "app": "test.test_drop.SumupContainerChecksum", "inputs": ["D"] }, { "oid": "F", "type": "plain", "storage": "memory", "producers": ["E"] }] roots = droputils.get_roots(pg_spec) self.assertEqual(2, len(roots)) self.assertListEqual(['A', 'B'], sorted(roots)) # The same as before but using dropdicts pg_spec_dropdicts = [dropdict(dropspec) for dropspec in pg_spec] roots = droputils.get_roots(pg_spec_dropdicts) self.assertEqual(2, len(roots)) self.assertListEqual(['A', 'B'], sorted(roots))
def sleepAndCopy(uid, **kwargs): dropSpec = dropdict({'oid':uid, 'type':'app', 'app':'test.graphsRepository.SleepAndCopyApp'}) dropSpec.update(kwargs) return dropSpec
def scpSpec(uid, **kwargs): dropSpec = dropdict({'oid':str(uid), 'type':'app', 'app':'dfms.apps.scp.ScpApp'}) dropSpec.update(kwargs) return dropSpec
def directorySpec(uid, **kwargs): dropSpec = dropdict({'oid':str(uid), 'type':'container', 'container':'dfms.drop.DirectoryContainer', 'node': 'localhost', 'island': 'localhost'}) dropSpec.update(kwargs) return dropSpec
def cleanSpec(uid, **kwargs): dropSpec = dropdict({'oid':str(uid), 'type':'app', 'app':'test.integrate.chiles.chilesdo.Clean'}) dropSpec.update(kwargs) return dropSpec
def fileDropSpec(uid, **kwargs): dropSpec = dropdict({'oid':str(uid), 'type':'plain', 'storage':'file', 'node': 'localhost', 'island': 'localhost'}) dropSpec.update(kwargs) return dropSpec
def casapyDockerAppSpec(uid, script): cmd = 'cd; ' + os.path.join(CASAPY, 'casapy') + ' --colors=NoColor --nologger --nogui -c "%s"' % (script) return dropdict({'oid':str(uid), 'type':'app', 'app':'dfms.apps.dockerapp.DockerApp', 'image':'dfms/casapy_centos7_dfms:0.1', 'command':cmd, 'user': '******', 'node': 'localhost', 'island': 'localhost'})
def socket(uid, port=1111): return dropdict({'oid':uid, 'type':'socket', 'port':port})
def memory(uid, **kwargs): dropSpec = dropdict({'oid':uid, 'type':'plain', 'storage':'memory'}) dropSpec.update(kwargs) return dropSpec
def fileSpec(uid, **kwargs): dropSpec = dropdict({'oid': str(uid), 'type': 'plain', 'storage': 'file'}) dropSpec.update(kwargs) return dropSpec