def partition(pgt, pip_name, num_partitions, num_islands, algo='metis'):
'''
Partitions the Physical Graph Template `pgt` with the algorithm `algo`
using `num_partitions` partitions.
'''
from dfms.dropmake.pg_generator import MySarkarPGTP, MetisPGTP
logger.info("Initialising PGTP %s", algo)
if algo == 'sarkar':
pgtp = MySarkarPGTP(pgt, num_partitions, merge_parts=True)
else:
pgtp = MetisPGTP(pgt, num_partitions, merge_parts=True)
del pgt
logger.info("PGTP initialised %s", algo)
logger.info("Starting to partition %s", pip_name)
pgtp.to_gojs_json(string_rep=False, visual=True)
pgt = pgtp.to_pg_spec([],
ret_str=False,
num_islands=num_islands,
tpl_nodes_len=num_partitions + num_islands)
logger.info("Partitioning completed for %s", pip_name)
return pgt
def test_mysarkar_pgtp_gen_pg(self):
lgnames = [
'lofar_std.json', 'test_grpby_gather.json', 'chiles_simple.json'
]
tgt_partnum = [15, 15, 10, 10, 5]
node_list = ['10.128.0.11', '10.128.0.12', '10.128.0.13']
for i, lgn in enumerate(lgnames):
fp = get_lg_fname(lgn)
lg = LG(fp)
drop_list = lg.unroll_to_tpl()
pgtp = MySarkarPGTP(drop_list, 3, merge_parts=True)
#pgtp.json
pgtp.to_gojs_json(visual=False)
pg_spec = pgtp.to_pg_spec(node_list)
def test_mysarkar_pgtp_gen_pg_island(self):
lgnames = [
'lofar_std.json', 'test_grpby_gather.json', 'chiles_simple.json'
]
node_list = [
'10.128.0.11', '10.128.0.12', '10.128.0.13', '10.128.0.14',
'10.128.0.15', '10.128.0.16'
]
for i, lgn in enumerate(lgnames):
fp = get_lg_fname(lgn)
lg = LG(fp)
drop_list = lg.unroll_to_tpl()
pgtp = MySarkarPGTP(drop_list, None, merge_parts=True)
pgtp.to_gojs_json(visual=False)
nb_islands = 2
#print(lgn)
try:
pgtp.merge_partitions(len(node_list) - nb_islands,
form_island=False)
except GPGTNoNeedMergeException as ge:
continue
pg_spec = pgtp.to_pg_spec(node_list, num_islands=nb_islands)
pgtp.result()
def gen_mysarkar_pgtp(lgfname,
pgt_dir,
num_islands=2,
cores_per_node=2,
print_result=False):
"""
Generate Physical Graph Template (Partition) using
MySarkar - A "somewhat greedy" scheudling algoritm
No real resource mapping is involved
"""
par_label = 'p'
stt = time.time()
lg = LG(lgfname)
drop_list = lg.unroll_to_tpl()
unroll_ett = time.time()
mpp = num_islands > 0
pgt = MySarkarPGTP(drop_list,
1,
par_label,
cores_per_node,
merge_parts=mpp)
if (mpp):
pgt.to_gojs_json(string_rep=False, visual=False)
pgt.merge_partitions(num_islands,
form_island=True,
island_type=1,
visual=False)
schedule_ett = time.time()
re_dict = pgt.result()
re_dict['unroll_time'] = '%.3f' % (unroll_ett - stt)
re_dict['schedule_time'] = '%.3f' % (schedule_ett - unroll_ett)
if (print_result):
part_info = ' - '.join(
['{0}:{1}'.format(k, v) for k, v in re_dict.items()])
print(part_info)
return re_dict
