def submit_single_graph(self, graph_id, algo="sarkar", deploy=False): lgn = lgnames[graph_id] fp = pkg_resources.resource_filename("dfms.dropmake", "web/{0}".format(lgn)) lg = LG(fp) drop_list = lg.unroll_to_tpl() # node_list = self.get_avail_hosts() node_list = self._dc.nodes() pgtp = MySarkarPGTP(drop_list, len(node_list), merge_parts=True) pgtp.json pg_spec = pgtp.to_pg_spec(node_list, ret_str=False) if self._output: with open(self._output, "w") as f: json.dump(pg_spec, f, indent=2) completed_uids = [x["oid"] for x in droputils.get_roots(pg_spec)] ssid = "{0}-{1}".format(lgn.split(".")[0], lg._session_id) self._dc.create_session(ssid) print "session created" self._dc.append_graph(ssid, pg_spec) print "graph appended" if deploy: ret = self._dc.deploy_session(ssid, completed_uids=completed_uids) print "session deployed" return ret
def partition(pgt, pip_name, num_partitions, num_islands, algo='metis'): ''' Partitions the Physical Graph Template `pgt` with the algorithm `algo` using `num_partitions` partitions. ''' from dfms.dropmake.pg_generator import MySarkarPGTP, MetisPGTP logger.info("Initialising PGTP %s", algo) if algo == 'sarkar': pgtp = MySarkarPGTP(pgt, num_partitions, merge_parts=True) else: pgtp = MetisPGTP(pgt, num_partitions, merge_parts=True) del pgt logger.info("PGTP initialised %s", algo) logger.info("Starting to partition %s", pip_name) pgtp.to_gojs_json(string_rep=False, visual=True) pgt = pgtp.to_pg_spec([], ret_str=False, num_islands=num_islands, tpl_nodes_len=num_partitions + num_islands) logger.info("Partitioning completed for %s", pip_name) return pgt
def test_mysarkar_pgtp_gen_pg(self): lgnames = ['lofar_std.json', 'chiles_two.json', 'test_grpby_gather.json', 'chiles_two_dev1.json', 'chiles_simple.json'] tgt_partnum = [15, 15, 10, 10, 5] node_list = ['10.128.0.11', '10.128.0.12', '10.128.0.13'] for i, lgn in enumerate(lgnames): fp = pkg_resources.resource_filename('dfms.dropmake', 'web/{0}'.format(lgn)) lg = LG(fp) drop_list = lg.unroll_to_tpl() pgtp = MySarkarPGTP(drop_list, 3, merge_parts=True) pgtp.json pg_spec = pgtp.to_pg_spec(node_list)
def produce_physical_graphs(self, graph_id, algo="sarkar", tgt="/tmp"): lgn = lgnames[graph_id] fp = pkg_resources.resource_filename("dfms.dropmake", "web/{0}".format(lgn)) lg = LG(fp) drop_list = lg.unroll_to_tpl() node_list = self._dc.nodes() # node_list = ['10.128.0.11', '10.128.0.14', '10.128.0.15', '10.128.0.16'] pgtp = MySarkarPGTP(drop_list, len(node_list), merge_parts=True) pgtp.json pg_spec = pgtp.to_pg_spec(node_list) with open("/{1}/sar_{0}_pgspec.json".format(lgn.split(".")[0], tgt), "w") as f: f.write(pg_spec)
def test_mysarkar_pgtp_gen_pg(self): lgnames = [ 'lofar_std.json', 'test_grpby_gather.json', 'chiles_simple.json' ] tgt_partnum = [15, 15, 10, 10, 5] node_list = ['10.128.0.11', '10.128.0.12', '10.128.0.13'] for i, lgn in enumerate(lgnames): fp = get_lg_fname(lgn) lg = LG(fp) drop_list = lg.unroll_to_tpl() pgtp = MySarkarPGTP(drop_list, 3, merge_parts=True) #pgtp.json pgtp.to_gojs_json(visual=False) pg_spec = pgtp.to_pg_spec(node_list)
def test_mysarkar_pgtp_gen_pg(self): lgnames = [ 'lofar_std.json', 'chiles_two.json', 'test_grpby_gather.json', 'chiles_two_dev1.json', 'chiles_simple.json' ] tgt_partnum = [15, 15, 10, 10, 5] node_list = ['10.128.0.11', '10.128.0.12', '10.128.0.13'] for i, lgn in enumerate(lgnames): fp = pkg_resources.resource_filename('dfms.dropmake', 'web/{0}'.format(lgn)) lg = LG(fp) drop_list = lg.unroll_to_tpl() pgtp = MySarkarPGTP(drop_list, 3, merge_parts=True) pgtp.json pg_spec = pgtp.to_pg_spec(node_list)
def test_mysarkar_pgtp_gen_pg_island(self): lgnames = [ 'lofar_std.json', 'test_grpby_gather.json', 'chiles_simple.json' ] node_list = [ '10.128.0.11', '10.128.0.12', '10.128.0.13', '10.128.0.14', '10.128.0.15', '10.128.0.16' ] for i, lgn in enumerate(lgnames): fp = get_lg_fname(lgn) lg = LG(fp) drop_list = lg.unroll_to_tpl() pgtp = MySarkarPGTP(drop_list, None, merge_parts=True) pgtp.to_gojs_json(visual=False) nb_islands = 2 #print(lgn) try: pgtp.merge_partitions(len(node_list) - nb_islands, form_island=False) except GPGTNoNeedMergeException as ge: continue pg_spec = pgtp.to_pg_spec(node_list, num_islands=nb_islands) pgtp.result()