def get_pg(opts, nms, dims):
"""Gets the Physical Graph that is eventually submitted to the cluster, if any"""
if not opts.logical_graph and not opts.physical_graph:
return []
num_nms = len(nms)
num_dims = len(dims)
if opts.logical_graph:
unrolled = pg_generator.unroll(opts.logical_graph, opts.ssid,
opts.zerorun, APPS[opts.app])
algo_params = parse_partition_algo_params(opts.algo_params)
pgt = pg_generator.partition(
unrolled,
opts.part_algo,
num_partitions=num_nms,
num_islands=num_dims,
**algo_params,
)
del unrolled # quickly dispose of potentially big object
else:
with open(opts.physical_graph, "rb") as pg_file:
pgt = json.load(pg_file)
# modify the PG as necessary
for modifier in opts.pg_modifiers.split(":"):
if modifier is not None and modifier != "":
modify_pg(pgt, modifier)
# Check which NMs are up and use only those form now on
nms = check_hosts(
nms,
NODE_DEFAULT_REST_PORT,
check_with_session=opts.check_with_session,
timeout=MM_WAIT_TIME,
retry=3,
)
LOGGER.info(
f"Mapping graph to available resources: nms {nms}, dims {dims}")
physical_graph = pg_generator.resource_map(pgt,
dims + nms,
num_islands=num_dims,
co_host_dim=opts.co_host_dim)
graph_name = os.path.basename(opts.log_dir)
graph_name = f"{graph_name.split('_')[0]}.json" # get just the graph name
with open(os.path.join(opts.log_dir, graph_name), "wt") as pg_file:
json.dump(physical_graph, pg_file)
return physical_graph
def unroll_and_partition_with_params(lg, algo_params_source):
# Get the 'test' parameter
# NB: the test parameter is a string, so convert to boolean
test = algo_params_source.get("test", "false")
test = test.lower() == "true"
# Based on 'test' parameter, decide whether to use a replacement app
app = "dlg.apps.simple.SleepApp" if test else None
# Unrolling LG to PGT.
pgt = init_pgt_unroll_repro_data(unroll(lg, app=app))
# Define partitioning parameters.
algo = algo_params_source.get("algo", "none")
num_partitions = algo_params_source.get("num_par", default=1, type=int)
num_islands = algo_params_source.get("num_islands", default=0, type=int)
par_label = algo_params_source.get("par_label", "Partition")
# Build a map with extra parameters, more specific to some par_algoithms.
algo_params = {}
for name, typ in ALGO_PARAMS:
if name in algo_params_source:
algo_params[name] = algo_params_source.get(name, type=typ)
reprodata = pgt.pop()
# Partition the PGT
pgt = partition(
pgt,
algo=algo,
num_partitions=num_partitions,
num_islands=num_islands,
partition_label=par_label,
show_gojs=True,
**algo_params,
)
pgt_spec = pgt.to_pg_spec(
[],
ret_str=False,
num_islands=num_islands,
tpl_nodes_len=num_partitions + num_islands,
)
pgt_spec.append(reprodata)
init_pgt_partition_repro_data(pgt_spec)
reprodata = pgt_spec.pop()
pgt.reprodata = reprodata
logger.info(reprodata)
return pgt
def get_pg(opts, num_node_managers, num_data_island_managers):
if not opts.logical_graph and not opts.physical_graph:
return []
if opts.logical_graph:
unrolled_graph = pg_generator.unroll(opts.logical_graph)
pgt = pg_generator.partition(
unrolled_graph,
algo="metis",
num_partitions=num_node_managers,
num_islands=num_data_island_managers,
)
del unrolled_graph
else:
with open(opts.physical_graph, "r", encoding="utf-8") as pg_file:
pgt = json.load(pg_file)
return pgt
def _create_pg(logical_graph, processing_block, node_managers,
data_island_manager, zero_cost_run):
logical_graph = pg_generator.fill(logical_graph,
processing_block.parameters)
unroll_kwargs = {}
if zero_cost_run:
unroll_kwargs['zerorun'] = True
unroll_kwargs['app'] = 'dlg.apps.simple.SleepApp'
physical_graph_template = pg_generator.unroll(logical_graph,
**unroll_kwargs)
physical_graph = pg_generator.partition(physical_graph_template,
'mysarkar',
num_partitions=len(node_managers),
num_islands=1)
physical_graph = pg_generator.resource_map(
physical_graph, [data_island_manager] + node_managers, num_islands=1)
return physical_graph