def distribute(
computation_graph: ComputationGraph,
agentsdef: Iterable[AgentDef],
hints: DistributionHints = None,
computation_memory: Callable[[ComputationNode], float] = None,
communication_load: Callable[[ComputationNode, str], float] = None,
timeout=None, # not used
) -> Distribution:
if computation_memory is None:
raise ImpossibleDistributionException("adhoc distribution requires "
"computation_memory functions")
mapping = defaultdict(lambda: [])
agents_capa = {a.name: a.capacity for a in agentsdef}
computations = computation_graph.node_names()
# as we're dealing with a secp modelled as a constraint graph,
# we only have actuator and pysical model variables.
# First, put each actuator variable on its agent
for agent in agentsdef:
for comp in computation_graph.node_names():
if agent.hosting_cost(comp) == 0:
mapping[agent.name].append(comp)
computations.remove(comp)
agents_capa[agent.name] -= computation_memory(
computation_graph.computation(comp))
if agents_capa[agent.name] < 0:
raise ImpossibleDistributionException(
f"Not enough capacity on {agent} to hosts actuator {comp}: {agents_capa[agent.name]}"
)
break
logger.info(f"Actuator variables on agents: {dict(mapping)}")
# We must now place physical model variable on an agent that host
# a variable it depends on.
# As physical models always depends on actuator variable,
# there must always be a computation it depends on that is already hosted.
for comp in computations:
footprint = computation_memory(computation_graph.computation(comp))
neighbors = computation_graph.neighbors(comp)
candidates = find_candidates(agents_capa, comp, footprint, mapping,
neighbors)
# Host the computation on the first agent and decrease its remaining capacity
selected = candidates[0][2]
mapping[selected].append(comp)
agents_capa[selected] -= footprint
return Distribution({a: list(mapping[a]) for a in mapping})
def distribute(computation_graph: ComputationGraph,
agentsdef: Iterable[AgentDef],
hints: DistributionHints = None,
computation_memory=None,
communication_load=None):
"""
Generate a distribution for the dcop.
:param computation_graph: a ComputationGraph
:param agentsdef: the agents definitions
:param hints: a DistributionHints
:param computation_memory: a function that takes a computation node as an
argument and return the memory footprint for this
:param link_communication: a function that takes a Link as an argument
and return the communication cost of this edge
"""
if computation_memory is None or communication_load is None:
raise ImpossibleDistributionException(
'LinearProg distribution requires '
'computation_memory and link_communication functions')
agents = list(agentsdef)
# In order to remove (latter on) distribution hints, we interpret
# hosting costs of 0 as a "must host" relationship
must_host = defaultdict(lambda: [])
for agent in agentsdef:
for comp in computation_graph.node_names():
if agent.hosting_cost(comp) == 0:
must_host[agent.name].append(comp)
logger.debug(f"Must host: {must_host}")
return factor_graph_lp_model(computation_graph, agents, must_host,
computation_memory, communication_load)
def distribute(
computation_graph: ComputationGraph,
agentsdef: Iterable[AgentDef],
hints=None,
computation_memory: Callable[[ComputationNode], float] = None,
communication_load: Callable[[ComputationNode, str], float] = None,
) -> Distribution:
"""
gh-cgdp distribution method.
Heuristic distribution baed on communication and hosting costs, while respecting
agent's capacities
Parameters
----------
computation_graph
agentsdef
hints
computation_memory
communication_load
Returns
-------
Distribution:
The distribution for the computation graph.
"""
# Place computations with hosting costs == 0
# For SECP, this assign actuators var and factor to the right device.
fixed_mapping = {}
for comp in computation_graph.node_names():
for agent in agentsdef:
if agent.hosting_cost(comp) == 0:
fixed_mapping[comp] = (
agent.name,
computation_memory(computation_graph.computation(comp)),
)
break
# Sort computation by footprint, but add a random element to avoid sorting on names
computations = [(computation_memory(n), n, None, random.random())
for n in computation_graph.nodes
if n.name not in fixed_mapping]
computations = sorted(computations,
key=lambda o: (o[0], o[3]),
reverse=True)
computations = [t[:-1] for t in computations]
logger.info("placing computations %s",
[(f, c.name) for f, c, _ in computations])
current_mapping = {} # Type: Dict[str, str]
i = 0
while len(current_mapping) != len(computations):
footprint, computation, candidates = computations[i]
logger.debug(
"Trying to place computation %s with footprint %s",
computation.name,
footprint,
)
# look for cancidiate agents for computation c
# TODO: keep a list of remaining capacities for agents ?
if candidates is None:
candidates = candidate_hosts(
computation,
footprint,
computations,
agentsdef,
communication_load,
current_mapping,
fixed_mapping,
)
computations[i] = footprint, computation, candidates
logger.debug("Candidates for computation %s : %s", computation.name,
candidates)
if not candidates:
if i == 0:
logger.error(
f"Cannot find a distribution, no candidate for computation {computation}\n"
f" current mapping: {current_mapping}")
raise ImpossibleDistributionException(
f"Impossible Distribution, no candidate for {computation}")
# no candidate : backtrack !
i -= 1
logger.info(
"No candidate for %s, backtrack placement "
"of computation %s (was on %s",
computation.name,
computations[i][1].name,
current_mapping[computations[i][1].name],
)
current_mapping.pop(computations[i][1].name)
# FIXME : eliminate selected agent for previous computation
else:
_, selected = candidates.pop()
current_mapping[computation.name] = selected.name
computations[i] = footprint, computation, candidates
logger.debug("Place computation %s on agent %s", computation.name,
selected.name)
i += 1
# Build the distribution for the mapping
agt_mapping = defaultdict(lambda: [])
for c, a in current_mapping.items():
agt_mapping[a].append(c)
for c, (a, _) in fixed_mapping.items():
agt_mapping[a].append(c)
dist = Distribution(agt_mapping)
return dist
def ilp_cgdp(
cg: ComputationGraph,
agentsdef: Iterable[AgentDef],
footprint: Callable[[str], float],
capacity: Callable[[str], float],
route: Callable[[str, str], float],
msg_load: Callable[[str, str], float],
hosting_cost: Callable[[str, str], float],
):
agt_names = [a.name for a in agentsdef]
pb = LpProblem("oilp_cgdp", LpMinimize)
# One binary variable xij for each (variable, agent) couple
xs = LpVariable.dict("x", (cg.node_names(), agt_names), cat=LpBinary)
# TODO: Do not create var for computation that are already assigned to an agent with hosting = 0 ?
# Force computation with hosting cost of 0 to be hosted on that agent.
# This makes the work much easier for glpk !
x_fixed_to_0 = []
x_fixed_to_1 = []
for agent in agentsdef:
for comp in cg.node_names():
assigned_agent = None
if agent.hosting_cost(comp) == 0:
pb += xs[(comp, agent.name)] == 1
x_fixed_to_1.append((comp, agent.name))
assigned_agent = agent.name
for other_agent in agentsdef:
if other_agent.name == assigned_agent:
continue
pb += xs[(comp, other_agent.name)] == 0
x_fixed_to_0.append((comp, other_agent.name))
logger.debug(
f"Setting binary varaibles to fixed computation {comp}")
# One binary variable for computations c1 and c2, and agent a1 and a2
betas = {}
count = 0
for a1, a2 in combinations(agt_names, 2):
# Only create variables for couple c1, c2 if there is an edge in the
# graph between these two computations.
for l in cg.links:
# As we support hypergraph, we may have more than 2 ends to a link
for c1, c2 in combinations(l.nodes, 2):
if (c1, a1, c2, a2) in betas:
continue
count += 2
b = LpVariable("b_{}_{}_{}_{}".format(c1, a1, c2, a2),
cat=LpBinary)
betas[(c1, a1, c2, a2)] = b
# Linearization constraints :
# a_ijmn <= x_im
# a_ijmn <= x_jn
if (c1, a1) in x_fixed_to_0 or (c2, a2) in x_fixed_to_0:
pb += b == 0
elif (c1, a1) in x_fixed_to_1:
pb += b == xs[(c2, a2)]
elif (c2, a2) in x_fixed_to_1:
pb += b == xs[(c1, a1)]
else:
pb += b <= xs[(c1, a1)]
pb += b <= xs[(c2, a2)]
pb += b >= xs[(c2, a2)] + xs[(c1, a1)] - 1
b = LpVariable("b_{}_{}_{}_{}".format(c1, a2, c2, a1),
cat=LpBinary)
if (c1, a2) in x_fixed_to_0 or (c2, a1) in x_fixed_to_0:
pb += b == 0
elif (c1, a2) in x_fixed_to_1:
pb += b == xs[(c2, a1)]
elif (c2, a1) in x_fixed_to_1:
pb += b == xs[(c1, a2)]
else:
betas[(c1, a2, c2, a1)] = b
pb += b <= xs[(c2, a1)]
pb += b <= xs[(c1, a2)]
pb += b >= xs[(c1, a2)] + xs[(c2, a1)] - 1
# Set objective: communication + hosting_cost
pb += (
_objective(xs, betas, route, msg_load, hosting_cost),
"Communication costs and prefs",
)
# Adding constraints:
# Constraints: Memory capacity for all agents.
for a in agt_names:
pb += (
lpSum([footprint(i) * xs[i, a]
for i in cg.node_names()]) <= capacity(a),
"Agent {} capacity".format(a),
)
# Constraints: all computations must be hosted.
for c in cg.node_names():
pb += (
lpSum([xs[c, a] for a in agt_names]) == 1,
"Computation {} hosted".format(c),
)
# solve using GLPK
status = pb.solve(
solver=GLPK_CMD(keepFiles=1, msg=False, options=["--pcost"]))
if status != LpStatusOptimal:
raise ImpossibleDistributionException("No possible optimal"
" distribution ")
logger.debug("GLPK cost : %s", pulp.value(pb.objective))
mapping = {}
for k in agt_names:
agt_computations = [
i for i, ka in xs if ka == k and pulp.value(xs[(i, ka)]) == 1
]
# print(k, ' -> ', agt_computations)
mapping[k] = agt_computations
return mapping