def submit_single_graph(self, graph_id, algo="sarkar", deploy=False):
lgn = lgnames[graph_id]
fp = pkg_resources.resource_filename("dfms.dropmake", "web/{0}".format(lgn))
lg = LG(fp)
drop_list = lg.unroll_to_tpl()
# node_list = self.get_avail_hosts()
node_list = self._dc.nodes()
pgtp = MySarkarPGTP(drop_list, len(node_list), merge_parts=True)
pgtp.json
pg_spec = pgtp.to_pg_spec(node_list, ret_str=False)
if self._output:
with open(self._output, "w") as f:
json.dump(pg_spec, f, indent=2)
completed_uids = [x["oid"] for x in droputils.get_roots(pg_spec)]
ssid = "{0}-{1}".format(lgn.split(".")[0], lg._session_id)
self._dc.create_session(ssid)
print "session created"
self._dc.append_graph(ssid, pg_spec)
print "graph appended"
if deploy:
ret = self._dc.deploy_session(ssid, completed_uids=completed_uids)
print "session deployed"
return ret
def partition(pgt, pip_name, num_partitions, num_islands, algo='metis'):
'''
Partitions the Physical Graph Template `pgt` with the algorithm `algo`
using `num_partitions` partitions.
'''
from dfms.dropmake.pg_generator import MySarkarPGTP, MetisPGTP
logger.info("Initialising PGTP %s", algo)
if algo == 'sarkar':
pgtp = MySarkarPGTP(pgt, num_partitions, merge_parts=True)
else:
pgtp = MetisPGTP(pgt, num_partitions, merge_parts=True)
del pgt
logger.info("PGTP initialised %s", algo)
logger.info("Starting to partition %s", pip_name)
pgtp.to_gojs_json(string_rep=False, visual=True)
pgt = pgtp.to_pg_spec([],
ret_str=False,
num_islands=num_islands,
tpl_nodes_len=num_partitions + num_islands)
logger.info("Partitioning completed for %s", pip_name)
return pgt
def test_mysarkar_pgtp_gen_pg(self):
lgnames = ['lofar_std.json', 'chiles_two.json', 'test_grpby_gather.json', 'chiles_two_dev1.json', 'chiles_simple.json']
tgt_partnum = [15, 15, 10, 10, 5]
node_list = ['10.128.0.11', '10.128.0.12', '10.128.0.13']
for i, lgn in enumerate(lgnames):
fp = pkg_resources.resource_filename('dfms.dropmake', 'web/{0}'.format(lgn))
lg = LG(fp)
drop_list = lg.unroll_to_tpl()
pgtp = MySarkarPGTP(drop_list, 3, merge_parts=True)
pgtp.json
pg_spec = pgtp.to_pg_spec(node_list)
def produce_physical_graphs(self, graph_id, algo="sarkar", tgt="/tmp"):
lgn = lgnames[graph_id]
fp = pkg_resources.resource_filename("dfms.dropmake", "web/{0}".format(lgn))
lg = LG(fp)
drop_list = lg.unroll_to_tpl()
node_list = self._dc.nodes()
# node_list = ['10.128.0.11', '10.128.0.14', '10.128.0.15', '10.128.0.16']
pgtp = MySarkarPGTP(drop_list, len(node_list), merge_parts=True)
pgtp.json
pg_spec = pgtp.to_pg_spec(node_list)
with open("/{1}/sar_{0}_pgspec.json".format(lgn.split(".")[0], tgt), "w") as f:
f.write(pg_spec)
def test_mysarkar_pgtp_gen_pg(self):
lgnames = [
'lofar_std.json', 'test_grpby_gather.json', 'chiles_simple.json'
]
tgt_partnum = [15, 15, 10, 10, 5]
node_list = ['10.128.0.11', '10.128.0.12', '10.128.0.13']
for i, lgn in enumerate(lgnames):
fp = get_lg_fname(lgn)
lg = LG(fp)
drop_list = lg.unroll_to_tpl()
pgtp = MySarkarPGTP(drop_list, 3, merge_parts=True)
#pgtp.json
pgtp.to_gojs_json(visual=False)
pg_spec = pgtp.to_pg_spec(node_list)
def test_mysarkar_pgtp_gen_pg(self):
lgnames = [
'lofar_std.json', 'chiles_two.json', 'test_grpby_gather.json',
'chiles_two_dev1.json', 'chiles_simple.json'
]
tgt_partnum = [15, 15, 10, 10, 5]
node_list = ['10.128.0.11', '10.128.0.12', '10.128.0.13']
for i, lgn in enumerate(lgnames):
fp = pkg_resources.resource_filename('dfms.dropmake',
'web/{0}'.format(lgn))
lg = LG(fp)
drop_list = lg.unroll_to_tpl()
pgtp = MySarkarPGTP(drop_list, 3, merge_parts=True)
pgtp.json
pg_spec = pgtp.to_pg_spec(node_list)
def test_mysarkar_pgtp_gen_pg_island(self):
lgnames = [
'lofar_std.json', 'test_grpby_gather.json', 'chiles_simple.json'
]
node_list = [
'10.128.0.11', '10.128.0.12', '10.128.0.13', '10.128.0.14',
'10.128.0.15', '10.128.0.16'
]
for i, lgn in enumerate(lgnames):
fp = get_lg_fname(lgn)
lg = LG(fp)
drop_list = lg.unroll_to_tpl()
pgtp = MySarkarPGTP(drop_list, None, merge_parts=True)
pgtp.to_gojs_json(visual=False)
nb_islands = 2
#print(lgn)
try:
pgtp.merge_partitions(len(node_list) - nb_islands,
form_island=False)
except GPGTNoNeedMergeException as ge:
continue
pg_spec = pgtp.to_pg_spec(node_list, num_islands=nb_islands)
pgtp.result()
