def distribution_cost(
distribution: Distribution,
computation_graph: ComputationGraph,
agentsdef: Iterable[AgentDef],
computation_memory: Callable[[ComputationNode], float],
communication_load: Callable[[ComputationNode, str], float],
) -> float:
route = route_fonc(agentsdef)
msg_load = msg_load_func(computation_graph, communication_load)
hosting_cost = hosting_cost_func(agentsdef)
comm = 0
agt_names = [a.name for a in agentsdef]
for l in computation_graph.links:
# As we support hypergraph, we may have more than 2 ends to a link
for c1, c2 in combinations(l.nodes, 2):
a1 = distribution.agent_for(c1)
a2 = distribution.agent_for(c2)
comm += route(a1, a2) * msg_load(c1, c2)
hosting = 0
for computation in computation_graph.nodes:
agent = distribution.agent_for(computation.name)
hosting += hosting_cost(agent, computation.name)
cost = RATIO_HOST_COMM * comm + (1 - RATIO_HOST_COMM) * hosting
return cost, comm, hosting
def test_is_hosted_several_computation(self):
d = Distribution({'a1': ['v1'], 'a2': ['v2']})
self.assertTrue(d.is_hosted(['v1', 'v2']))
self.assertTrue(d.is_hosted(['v2', 'v1']))
self.assertFalse(d.is_hosted(['v3']))
self.assertFalse(d.is_hosted(['v1', 'v3']))
self.assertFalse(d.is_hosted(['v3', 'v2']))
def test_dist(self):
d = Distribution({'a1': ['v1'], 'a2': ['v2']})
self.assertEqual(len(d.computations_hosted('a1')), 1)
self.assertEqual(len(d.computations_hosted('a2')), 1)
self.assertIn('v1', d.computations_hosted('a1'))
self.assertIn('v2', d.computations_hosted('a2'))
self.assertEqual(d.agent_for('v1'), 'a1')
self.assertEqual(d.agent_for('v2'), 'a2')
def distribute(computation_graph: ComputationGraph,
agentsdef: Iterable[AgentDef],
hints: DistributionHints = None,
computation_memory=None,
communication_load=None) -> Distribution:
"""
Simplistic distribution method: each computation is hosted on agent
agent and each agent host a single computation.
Agent capacity is not considered.
Raises an ImpossibleDistributionException
:param computation_graph:
:param agentsdef: AgntsDef object containing the list of agent, there must
be at least as many agents as computations
:param hints DistributionHints
:return: a distribution a dict {agent_name: [ var_name, ...]}
"""
agents = list(agentsdef)
if len(agents) < len(computation_graph.nodes):
raise ImpossibleDistributionException(
'Not enough agents for one agent for each computation : {} < {}'.
format(len(agents), len(computation_graph.nodes)))
agent_names = [a.name for a in agents]
distribution = defaultdict(lambda: list())
for n, a in zip(computation_graph.nodes, agent_names):
distribution[a].append(n.name)
return Distribution(distribution)
def distribute(
computation_graph: ComputationsFactorGraph,
agentsdef: Iterable[AgentDef],
hints=None,
computation_memory: Callable[[ComputationNode], float] = None,
communication_load: Callable[[ComputationNode, str], float] = None,
timeout=600, # Max 10 min
) -> Distribution:
if computation_memory is None or communication_load is None:
raise ImpossibleDistributionException(
"oilp_secp_fgdp distribution requires "
"computation_memory and link_communication functions"
)
mapping = defaultdict(lambda: [])
agents_capa = {a.name: a.capacity for a in agentsdef}
variable_computations, factor_computations = [], []
for comp in computation_graph.nodes:
if isinstance(comp, VariableComputationNode):
variable_computations.append(comp.name)
elif isinstance(comp, FactorComputationNode):
factor_computations.append(comp.name)
else:
raise ImpossibleDistributionException(
f"Error: {comp} is neither a factor nor a variable computation"
)
# actuators variables and cost factor on the corresponding agent:
for variable in variable_computations[:]:
for agent in agentsdef:
if agent.hosting_cost(variable) == 0:
# Found an actuator variable, host it on the agent
mapping[agent.name].append(variable)
variable_computations.remove(variable)
agents_capa[agent.name] -= computation_memory(
computation_graph.computation(variable)
)
# search for the cost factor, if any, and host it on the same agent.
for factor in factor_computations[:]:
if f"c_{variable}" == factor:
mapping[agent.name].append(factor)
factor_computations.remove(factor)
agents_capa[agent.name] -= computation_memory(
computation_graph.computation(factor)
)
if agents_capa[agent.name] < 0:
raise ImpossibleDistributionException(
f"Not enough capacity on {agent} to hosts actuator {variable}: {agents_capa[agent.name]}"
)
break
logger.info(f"Actuator variables - agents: {dict(mapping)}")
logger.info(f"Remaining capacity: {dict(agents_capa)}")
return fg_secp_ilp(
computation_graph,
agentsdef,
Distribution(mapping),
computation_memory,
communication_load,
)
def distribute(computation_graph: ComputationGraph,
agentsdef: Iterable[AgentDef],
hints: DistributionHints=None,
computation_memory=None,
communication_load=None) -> Distribution:
"""
Generate a distribution for the given computation graph.
:param computation_graph: a ComputationGraph
:param agentsdef: agents' definitions
:param hints: a DistributionHints
:param computation_memory: a function that takes a computation node and its
Link node as arguments and return the memory footprint for this node
:param communication_load: a function that takes a Link as an argument
and return the communication cost of this edge
"""
footprint = footprint_fonc(computation_graph, computation_memory)
capacity = capacity_fonc(agentsdef)
route = route_fonc(agentsdef)
msg_load = msg_load_func(computation_graph, communication_load)
hosting_cost = hosting_cost_func(agentsdef)
mapping = lp_model(computation_graph, agentsdef, footprint, capacity, route,
msg_load, hosting_cost)
dist = Distribution(mapping)
return dist
def load_dist(dist_str: str) -> Distribution:
loaded = yaml.load(dist_str)
if 'distribution' not in loaded:
raise ValueError('Invalid distribution file')
loaded_dist = loaded['distribution']
return Distribution(loaded_dist)
def distribution_cost(
distribution: Distribution, computation_graph: ComputationGraph,
agentsdef: Iterable[AgentDef],
computation_memory: Callable[[ComputationNode], float],
communication_load: Callable[[ComputationNode, str], float]) -> float:
"""
Compute the cost for a distribution.
In this model, the cost only includes the communication costs based on message size.
Parameters
----------
distribution
computation_graph
agentsdef
computation_memory
communication_load
Returns
-------
"""
# No hosting and route cost here, as this distribution only takes message size
# into account:
# route = route_fonc(agentsdef)
# hosting_cost = hosting_cost_func(agentsdef)
comm = 0
agt_names = [a.name for a in agentsdef]
for l in computation_graph.links:
# As we support hypergraph, we may have more than 2 ends to a link
for c1, c2 in combinations(l.nodes, 2):
if distribution.agent_for(c1) != distribution.agent_for(c2):
edge_cost = communication_load(
computation_graph.computation(c1), c2)
logger.debug(f"edge cost between {c1} and {c2} : {edge_cost}")
comm += edge_cost
else:
logger.debug(
f"On same agent, no edge cost between {c1} and {c2}")
# This distribution model only takes communication cost into account.
# cost = RATIO_HOST_COMM * comm + (1-RATIO_HOST_COMM) * hosting
return comm, comm, 0
def distribution_cost(
distribution: Distribution,
computation_graph: ComputationsFactorGraph,
agentsdef: Iterable[AgentDef],
computation_memory: Callable[[ComputationNode], float],
communication_load: Callable[[ComputationNode, str], float],
) -> float:
"""
Compute the cost of the distribution.
Only takes communication costs into account (no hosting nor route costs).
Parameters
----------
distribution
computation_graph
agentsdef
computation_memory
communication_load
Returns
-------
"""
comm = 0
agt_names = [a.name for a in agentsdef]
for l in computation_graph.links:
# As we support hypergraph, we may have more than 2 ends to a link
for c1, c2 in combinations(l.nodes, 2):
if distribution.agent_for(c1) != distribution.agent_for(c2):
edge_cost = communication_load(
computation_graph.computation(c1), c2)
logger.debug(f"edge cost between {c1} and {c2} : {edge_cost}")
comm += edge_cost
else:
logger.debug(
f"On same agent, no edge cost between {c1} and {c2}")
# This distribution model only takes communication cost into account.
# cost = RATIO_HOST_COMM * comm + (1-RATIO_HOST_COMM) * hosting
return comm, comm, 0
def test_host_on_agent(self):
d = Distribution({'a1': ['v1', 'v2'], 'a2': ['v3']})
d.host_on_agent('a1', ['v4'])
self.assertEqual(d.agent_for('v4'), 'a1')
self.assertEqual(d.agent_for('v1'), 'a1')
self.assertIn('v4', d.computations_hosted('a1'))
self.assertIn('v1', d.computations_hosted('a1'))
self.assertIn('v2', d.computations_hosted('a1'))
def distribute(
computation_graph: ComputationGraph,
agentsdef: Iterable[AgentDef],
hints: DistributionHints = None,
computation_memory: Callable[[ComputationNode], float] = None,
communication_load: Callable[[ComputationNode, str], float] = None,
timeout=None, # not used
) -> Distribution:
if computation_memory is None:
raise ImpossibleDistributionException("adhoc distribution requires "
"computation_memory functions")
mapping = defaultdict(lambda: [])
agents_capa = {a.name: a.capacity for a in agentsdef}
computations = computation_graph.node_names()
# as we're dealing with a secp modelled as a constraint graph,
# we only have actuator and pysical model variables.
# First, put each actuator variable on its agent
for agent in agentsdef:
for comp in computation_graph.node_names():
if agent.hosting_cost(comp) == 0:
mapping[agent.name].append(comp)
computations.remove(comp)
agents_capa[agent.name] -= computation_memory(
computation_graph.computation(comp))
if agents_capa[agent.name] < 0:
raise ImpossibleDistributionException(
f"Not enough capacity on {agent} to hosts actuator {comp}: {agents_capa[agent.name]}"
)
break
logger.info(f"Actuator variables on agents: {dict(mapping)}")
# We must now place physical model variable on an agent that host
# a variable it depends on.
# As physical models always depends on actuator variable,
# there must always be a computation it depends on that is already hosted.
for comp in computations:
footprint = computation_memory(computation_graph.computation(comp))
neighbors = computation_graph.neighbors(comp)
candidates = find_candidates(agents_capa, comp, footprint, mapping,
neighbors)
# Host the computation on the first agent and decrease its remaining capacity
selected = candidates[0][2]
mapping[selected].append(comp)
agents_capa[selected] -= footprint
return Distribution({a: list(mapping[a]) for a in mapping})
def distribute(computation_graph: ComputationGraph,
agentsdef: Iterable[AgentDef],
hints: DistributionHints = None,
computation_memory=None,
communication_load=None,
timeout=None) -> Distribution:
"""
Simplistic distribution method: each computation is hosted on agent
agent and each agent host a single computation.
Agent capacity is not considered.
Raises an ImpossibleDistributionException
Parameters
----------
computation_graph: a ComputationGraph
the computation graph containing the computation that must be
distributed
agentsdef: iterable of AgentDef objects
The definition of the agents the computation will be assigned to.
There **must** be at least as many agents as computations.
hints:
Not used by the ``oneagent`` distribution method.
computation_memory:
Not used by the ``oneagent`` distribution method.
computation_memory:
Not used by the ``oneagent`` distribution method.
Returns
-------
distribution: Distribution
A Distribution object containing the mapping form agents to
computations.
"""
agents = list(agentsdef)
if len(agents) < len(computation_graph.nodes):
raise ImpossibleDistributionException(
'Not enough agents for one agent for each computation : {} < {}'.
format(len(agents), len(computation_graph.nodes)))
agent_names = [a.name for a in agents]
distribution = defaultdict(lambda: list())
for n, a in zip(computation_graph.nodes, agent_names):
distribution[a].append(n.name)
return Distribution(distribution)
def distribute(
computation_graph: ComputationConstraintsHyperGraph,
agentsdef: Iterable[AgentDef],
hints=None,
computation_memory: Callable[[ComputationNode], float] = None,
communication_load: Callable[[ComputationNode, str], float] = None,
timeout=600, # Max 10 min
) -> Distribution:
if computation_memory is None or communication_load is None:
raise ImpossibleDistributionException(
"oilp_secp_cgdp distribution requires "
"computation_memory and link_communication functions")
mapping = defaultdict(lambda: [])
agents_capa = {a.name: a.capacity for a in agentsdef}
computations = computation_graph.node_names()
# as we're dealing with a secp modelled as a constraint graph,
# we only have actuator and pysical model variables.
# First, put each actuator variable on its agent
for agent in agentsdef:
for comp in computation_graph.node_names():
if agent.hosting_cost(comp) == 0:
mapping[agent.name].append(comp)
computations.remove(comp)
agents_capa[agent.name] -= computation_memory(
computation_graph.computation(comp))
if agents_capa[agent.name] < 0:
raise ImpossibleDistributionException(
f"Not enough capacity on {agent} to hosts actuator {comp}: {agents_capa[agent.name]}"
)
logger.info(f"Actuator variables on agents: {dict(mapping)}")
logger.info(f"Remaining capacity: {dict(agents_capa)}")
return cg_secp_ilp(
computation_graph,
agentsdef,
Distribution(mapping),
computation_memory,
communication_load,
)
def distribute(
computation_graph: ComputationGraph,
agentsdef: Iterable[AgentDef],
hints: DistributionHints = None,
computation_memory=None,
communication_load=None,
) -> Distribution:
"""
Parameters
----------
computation_graph
agentsdef
hints
computation_memory
communication_load
Returns
-------
"""
footprint_f = footprint_fonc(computation_graph, computation_memory)
capacity_f = capacity_fonc(agentsdef)
route_f = route_fonc(agentsdef)
msg_load_f = msg_load_func(computation_graph, communication_load)
hosting_cost_f = hosting_cost_func(agentsdef)
return Distribution(
ilp_cgdp(
computation_graph,
agentsdef,
footprint_f,
capacity_f,
route_f,
msg_load_f,
hosting_cost_f,
))
def _dump_repair_metrics(self, repair_status, repair_duration ):
# Dump current distribution
dist = {a: self.discovery.agent_computations(a)
for a in self.discovery.agents()}
result = {
'inputs': {
'dist_algo': 'repair',
},
"duration": repair_duration,
'distribution': dist,
"metrics": self.repair_metrics,
"status": repair_status
}
try:
cost, comm, hosting = gh_cgdp.distribution_cost(
Distribution(dist),
self.graph,
self._dcop.agents.values(), # AgentDef s
computation_memory=self._algo_module.computation_memory,
communication_load=self._algo_module.communication_load,
)
result["cost"] = cost
result["communication_cost"] = comm
result["hosting_cost"] = hosting
except Exception as e:
self.logger.error("Could not distribute ")
cost, comm, hosting = None, None, None
result["cost"] = None
result["communication_cost"] = None
result["hosting_cost"] = None
result["cost_error"] = str(e)
f_name = 'evtdist_{}.yaml'.format(self.dist_count)
with open(f_name, mode='w', encoding='utf-8') as f:
f.write(yaml.dump(result))
def fg_secp_ilp(
cg: ComputationsFactorGraph,
agents: List[AgentDef],
already_assigned: Distribution,
computation_memory: Callable[[ComputationNode], float],
communication_load: Callable[[ComputationNode, str], float],
) -> Distribution:
variables = [n for n in cg.nodes if n.type == "VariableComputation"]
factors = [n for n in cg.nodes if n.type == "FactorComputation"]
agents = list(agents)
agents_names = [a.name for a in agents]
# Only keep computations for which we actually need to find an agent.
vars_to_host = [
v.name for v in variables
if not already_assigned.has_computation(v.name)
]
facs_to_host = [
f.name for f in factors if not already_assigned.has_computation(f.name)
]
# x_i^k : binary variable indicating if var x_i is hosted on agent a_k.
xs = _build_xs_binvar(vars_to_host, agents_names)
# f_j^k : binary variable indicating if factor f_j is hosted on agent a_k.
fs = _build_fs_binvar(facs_to_host, agents_names)
# alpha_ijk : binary variable indicating if x_i and f_j are both on a_k.
alphas = _build_alphaijk_binvars(cg, agents_names)
logger.debug(f"alpha_ijk {alphas}")
# LP problem with objective function (total communication cost).
pb = LpProblem("distribution", LpMinimize)
pb += (
secp_dist_objective_function(cg, communication_load, alphas,
agents_names),
"Communication costs",
)
# Constraints.
# All variable computations must be hosted:
for i in vars_to_host:
pb += (
lpSum([xs[(i, k)] for k in agents_names]) == 1,
"var {} is hosted".format(i),
)
# All factor computations must be hosted:
for j in facs_to_host:
pb += (
lpSum([fs[(j, k)] for k in agents_names]) == 1,
"factor {} is hosted".format(j),
)
# Each agent must host at least one computation:
# We only need this constraints for agents that do not already host a
# computation:
empty_agents = [
a for a in agents_names if not already_assigned.computations_hosted(a)
]
for k in empty_agents:
pb += (
lpSum([xs[(i, k)] for i in vars_to_host]) +
lpSum([fs[(j, k)] for j in facs_to_host]) >= 1,
"atleastone {}".format(k),
)
# Memory capacity constraint for agents
for a in agents:
# Decrease capacity for already hosted computations
capacity = a.capacity - sum([
secp_computation_memory_in_cg(c, cg, computation_memory)
for c in already_assigned.computations_hosted(a.name)
])
pb += (
lpSum([
secp_computation_memory_in_cg(i, cg, computation_memory) * xs[
(i, a.name)] for i in vars_to_host
]) + lpSum([
secp_computation_memory_in_cg(j, cg, computation_memory) * fs[
(j, a.name)] for j in facs_to_host
]) <= capacity,
"memory {}".format(a.name),
)
# Linearization constraints for alpha_ijk.
for link in cg.links:
i, j = link.variable_node, link.factor_node
for k in agents_names:
if i in vars_to_host and j in facs_to_host:
pb += alphas[((i, j), k)] <= xs[(i, k)], "lin1 {}{}{}".format(
i, j, k)
pb += alphas[((i, j), k)] <= fs[(j, k)], "lin2 {}{}{}".format(
i, j, k)
pb += (
alphas[((i, j), k)] >= xs[(i, k)] + fs[(j, k)] - 1,
"lin3 {}{}{}".format(i, j, k),
)
elif i in vars_to_host and j not in facs_to_host:
# Var is free, factor is already hosted
if already_assigned.agent_for(j) == k:
pb += alphas[((i, j), k)] == xs[(i, k)]
else:
pb += alphas[((i, j), k)] == 0
elif i not in vars_to_host and j in facs_to_host:
# if i is not in vars_vars_to_host, it means that it's a
# computation that is already hosted (from hints)
if already_assigned.agent_for(i) == k:
pb += alphas[((i, j), k)] == fs[(j, k)]
else:
pb += alphas[((i, j), k)] == 0
else:
# i and j are both alredy hosted
if (already_assigned.agent_for(i) == k
and already_assigned.agent_for(j) == k):
pb += alphas[((i, j), k)] == 1
else:
pb += alphas[((i, j), k)] == 0
# Now solve our LP
# status = pb.solve(GLPK_CMD())
# status = pb.solve(GLPK_CMD(mip=1))
# status = pb.solve(GLPK_CMD(mip=0, keepFiles=1,
# options=['--simplex', '--interior']))
status = pb.solve(GLPK_CMD(keepFiles=0, msg=False, options=["--pcost"]))
if status != LpStatusOptimal:
raise ImpossibleDistributionException("No possible optimal"
" distribution ")
else:
logger.debug("GLPK cost : %s", pulp.value(pb.objective))
comp_dist = already_assigned
for k in agents_names:
agt_vars = [
i for i, ka in xs if ka == k and pulp.value(xs[(i, ka)]) == 1
]
comp_dist.host_on_agent(k, agt_vars)
agt_rels = [
j for j, ka in fs if ka == k and pulp.value(fs[(j, ka)]) == 1
]
comp_dist.host_on_agent(k, agt_rels)
return comp_dist
def factor_graph_lp_model(cg: ComputationsFactorGraph,
agents: List[AgentDef],
must_host: Dict[str, List],
computation_memory=None,
communication_load=None):
"""
To distribute we need:
* com : the communication cost of an edge between a var and a fact
* mem_var : the memory footprint of a variable computation
* mem_fac : the memory footprint of a factor computation
These function depends on the algorithm.
Here
* mem_var and mem_fac are given by the computation_memory method.
* com is given by computation_memory
:return:
"""
variables = [n for n in cg.nodes if n.type == 'VariableComputation']
factors = [n for n in cg.nodes if n.type == 'FactorComputation']
agents = list(agents)
agents_names = [a.name for a in agents]
fixed_dist = Distribution(must_host)
# Only keep computations for which we actually need to find an agent.
vars_to_host = [
v.name for v in variables if not fixed_dist.has_computation(v.name)
]
facs_to_host = [
f.name for f in factors if not fixed_dist.has_computation(f.name)
]
# x_i^k : binary variable indicating if var x_i is hosted on agent a_k.
xs = _build_xs_binvar(vars_to_host, agents_names)
# f_j^k : binary variable indicating if factor f_j is hosted on agent a_k.
fs = _build_fs_binvar(facs_to_host, agents_names)
# alpha_ijk : binary variable indicating if x_i and f_j are both on a_k.
alphas = _build_alphaijk_binvars(cg, agents_names)
# LP problem with objective function (total communication cost).
pb = LpProblem('distribution', LpMinimize)
pb += _objective_function(cg, communication_load, alphas,
agents_names), 'Communication costs'
# Constraints.
# All variable computations must be hosted:
for i in vars_to_host:
pb += lpSum([xs[(i, k)] for k in agents_names]) == 1, \
'var {} is hosted'.format(i)
# All factor computations must be hosted:
for j in facs_to_host:
pb += lpSum([fs[(j, k)] for k in agents_names]) == 1, \
'factor {} is hosted'.format(j)
# Each agent must host at least one computation:
# We only need this constraints for agents that do not already host a
# computation:
empty_agents = [a for a in agents_names if not must_host[a]]
for k in empty_agents:
pb += lpSum([xs[(i, k)] for i in vars_to_host]) + \
lpSum([fs[(j, k)] for j in facs_to_host]) >= 1, \
'atleastone {}'.format(k)
# Memory capacity constraint for agents
for a in agents:
# Decrease capacity for already hosted computations
capacity = a.capacity - \
sum([_computation_memory_in_cg(c, cg, computation_memory)
for c in must_host[a.name]])
pb += lpSum([_computation_memory_in_cg(i, cg, computation_memory) *
xs[(i, a.name)] for i in vars_to_host]) \
+ lpSum([_computation_memory_in_cg(j, cg, computation_memory) *
fs[(j, a.name)] for j in facs_to_host]) <= capacity, \
'memory {}'.format(a.name)
# Linearization constraints for alpha_ijk.
for link in cg.links:
i, j = link.variable_node, link.factor_node
for k in agents_names:
if i in vars_to_host and j in facs_to_host:
pb += alphas[((i, j), k)] <= xs[(i, k)], \
'lin1 {}{}{}'.format(i, j, k)
pb += alphas[((i, j), k)] <= fs[(j, k)], \
'lin2 {}{}{}'.format(i, j, k)
pb += alphas[((i, j), k)] >= xs[(i, k)] + fs[(j, k)] - 1, \
'lin3 {}{}{}'.format(i, j, k)
elif i in vars_to_host and j not in facs_to_host:
# Var is free, factor is already hosted
if fixed_dist.agent_for(j) == k:
pb += alphas[((i, j), k)] == xs[(i, k)]
else:
pb += alphas[((i, j), k)] == 0
elif i not in vars_to_host and j in facs_to_host:
# if i is not in vars_vars_to_host, it means that it's a
# computation that is already hosted (from hints)
if fixed_dist.agent_for(i) == k:
pb += alphas[((i, j), k)] == fs[(j, k)]
else:
pb += alphas[((i, j), k)] == 0
else:
# i and j are both alredy hosted
if fixed_dist.agent_for(i) == k and fixed_dist.agent_for(j) \
== k:
pb += alphas[((i, j), k)] == 1
else:
pb += alphas[((i, j), k)] == 0
# Now solve our LP
# status = pb.solve(GLPK_CMD())
# status = pb.solve(GLPK_CMD(mip=1))
# status = pb.solve(GLPK_CMD(mip=0, keepFiles=1,
# options=['--simplex', '--interior']))
status = pb.solve(GLPK_CMD(keepFiles=0, msg=False, options=['--pcost']))
if status != LpStatusOptimal:
raise ImpossibleDistributionException("No possible optimal"
" distribution ")
else:
logger.debug('GLPK cost : %s', value(pb.objective))
comp_dist = fixed_dist
for k in agents_names:
agt_vars = [
i for i, ka in xs if ka == k and value(xs[(i, ka)]) == 1
]
comp_dist.host_on_agent(k, agt_vars)
agt_rels = [
j for j, ka in fs if ka == k and value(fs[(j, ka)]) == 1
]
comp_dist.host_on_agent(k, agt_rels)
return comp_dist
def _distribute_try(computation_graph: ComputationGraph,
agents: Iterable[AgentDef],
hints: DistributionHints = None,
computation_memory=None,
communication_load=None,
attempt=0):
agents_capa = {a.name: a.capacity for a in agents}
# The distribution methods depends on the order used to process the node,
# we shuffle them to test a new configuration when retry a distribution
# after a failure
nodes = list(computation_graph.nodes)
shuffle(nodes)
mapping = defaultdict(set)
var_hosted = {}
# Distribute owned computation variable on the corresponding agent.
# For dcop build from an secp, this is the same thing as deploying the
# light variable on the light devices, as we were doing before.
for a in agents_capa:
for c in hints.must_host(a):
mapping[a].add(c)
var_hosted.update({c: a})
agents_capa[a] -= computation_memory(
computation_graph.computation(c))
# First mimic original secp adhoc behavior
for n in nodes:
if n.name in var_hosted:
continue
hostwith = hints.host_with(n.name)
# secp models have a constraint that should be hosted on the same
# agent than the variable of the model
if len(hostwith) == 1 and n.type == 'FactorComputation' and \
computation_graph.computation(hostwith[0]).type \
== 'VariableComputation':
dependent_var = [v.name for v in n.factor.dimensions]
candidates = [
a for a in agents_capa
if len(set(mapping[a]).intersection(dependent_var)) > 0
]
candidates.sort(key=lambda x: len(mapping[a]))
if candidates:
selected = candidates[0]
else:
selected = choice(list(agents_capa.keys()))
mapping[selected].update({n.name, hostwith[0]})
var_hosted[n.name] = selected
var_hosted[hostwith[0]] = selected
agents_capa[selected] -= computation_memory(n)
for n in nodes:
if n.name in var_hosted:
continue
footprint = computation_memory(n)
# Candidates : hints only with enough capacity
candidates = [(agents_capa[a], a) for a in hints.host_with(n.name)
if agents_capa[a] > footprint]
# If no hinted agents has enough capacity, fall back to all agents
if not candidates:
candidates = [(c, a) for a, c in agents_capa.items()
if c > footprint]
# Select the candidate that is already hosting the highest
# number of computations sharing a link with this one.
scores = []
for capacity, a in candidates:
count = 0
for l in computation_graph.links_for_node(n.name):
count += len([None for l_n in l.nodes if l_n in mapping[a]])
# The tuple is in this order so that we sort by score first,
# and then by available capacity.
scores.append((count, capacity, a))
scores.sort(reverse=True)
if scores:
selected = scores[0][2]
agents_capa[selected] -= footprint
else:
# Retry 3 times in case of failure, the nodes will be shuffled
# every time, increasing the probability to find a feasible
# distribution.
if attempt > 2:
raise ImpossibleDistributionException(
'Could not find feasible distribution after {} '
'attempts'.format(attempt))
else:
_distribute_try(computation_graph, agents, hints,
computation_memory, computation_graph,
attempt + 1)
mapping[selected].update({n.name})
var_hosted[n.name] = selected
return Distribution({a: list(mapping[a]) for a in mapping})
def distribute(
computation_graph: ComputationsFactorGraph,
agentsdef: Iterable[AgentDef],
hints: DistributionHints = None,
computation_memory: Callable[[ComputationNode], float] = None,
communication_load: Callable[[ComputationNode, str], float] = None,
timeout=None, # not used
) -> Distribution:
if computation_memory is None:
raise ImpossibleDistributionException("adhoc distribution requires "
"computation_memory functions")
# as we're dealing with a secp modelled as a factor graph, we have computations for
# actuator and physical model variables, rules and physical model factors.
mapping = defaultdict(lambda: [])
agents_capa = {a.name: a.capacity for a in agentsdef}
variable_computations = []
factor_computations = []
for comp in computation_graph.nodes:
if isinstance(comp, VariableComputationNode):
variable_computations.append(comp.name)
elif isinstance(comp, FactorComputationNode):
factor_computations.append(comp.name)
else:
raise ImpossibleDistributionException(
f"Error: {comp} is neither a factor nor a variable computation"
)
# First, put each actuator variable and cost factor on its agent
for variable in variable_computations[:]:
for agent in agentsdef:
if agent.hosting_cost(variable) == 0:
# Found an actuator variable, host it on the agent
mapping[agent.name].append(variable)
variable_computations.remove(variable)
agents_capa[agent.name] -= computation_memory(
computation_graph.computation(variable))
# search for the cost factor, if any, and host it on the same agent.
for factor in factor_computations[:]:
if f"c_{variable}" == factor:
mapping[agent.name].append(factor)
factor_computations.remove(factor)
agents_capa[agent.name] -= computation_memory(
computation_graph.computation(factor))
if agents_capa[agent.name] < 0:
raise ImpossibleDistributionException(
f"Not enough capacity on {agent} to hosts actuator {variable}: {agents_capa[agent.name]}"
)
break
logger.info(f"Actuator computations - agents: {dict(mapping)}")
logger.info(f"Remaining capacity: {dict(agents_capa)}")
# now find computations for physical models and variables variables.
# * all remaining variables are model variables
# * physical model factor computation names contain the name of the variable
model_variables = variable_computations
models = []
for model_var in model_variables:
for fact in factor_computations:
if f"c_{model_var}" == fact:
models.append((model_var, fact))
factor_computations.remove(fact)
# All remaining factor ar rule factors
rule_factors = factor_computations
logger.debug(f"Physical models: {models}")
logger.debug(f"Rules: {rule_factors}")
# Now place models
for model_var, model_fac in models:
footprint = computation_memory(
computation_graph.computation(model_fac)) + computation_memory(
computation_graph.computation(model_var))
neighbors = computation_graph.neighbors(model_fac)
candidates = find_candidates(agents_capa, model_fac, footprint,
mapping, neighbors)
# Host the model on the first agent and decrease its remaining capacity
selected = candidates[0][2]
mapping[selected].append(model_var)
mapping[selected].append(model_fac)
agents_capa[selected] -= footprint
logger.debug(f"All models hosted: {dict(mapping)}")
logger.debug(f"Remaining capacity: {agents_capa}")
# And rules at last:
for rule_fac in rule_factors:
footprint = computation_memory(computation_graph.computation(rule_fac))
neighbors = computation_graph.neighbors(rule_fac)
candidates = find_candidates(agents_capa, rule_fac, footprint, mapping,
neighbors)
# Host the computation on the first agent and decrease its remaining capacity
selected = candidates[0][2]
mapping[selected].append(rule_fac)
agents_capa[selected] -= footprint
return Distribution({a: list(mapping[a]) for a in mapping})
def distribute(
computation_graph: ComputationGraph,
agentsdef: Iterable[AgentDef],
hints=None,
computation_memory: Callable[[ComputationNode], float] = None,
communication_load: Callable[[ComputationNode, str], float] = None,
) -> Distribution:
"""
gh-cgdp distribution method.
Heuristic distribution baed on communication and hosting costs, while respecting
agent's capacities
Parameters
----------
computation_graph
agentsdef
hints
computation_memory
communication_load
Returns
-------
Distribution:
The distribution for the computation graph.
"""
# Place computations with hosting costs == 0
# For SECP, this assign actuators var and factor to the right device.
fixed_mapping = {}
for comp in computation_graph.node_names():
for agent in agentsdef:
if agent.hosting_cost(comp) == 0:
fixed_mapping[comp] = (
agent.name,
computation_memory(computation_graph.computation(comp)),
)
break
# Sort computation by footprint, but add a random element to avoid sorting on names
computations = [(computation_memory(n), n, None, random.random())
for n in computation_graph.nodes
if n.name not in fixed_mapping]
computations = sorted(computations,
key=lambda o: (o[0], o[3]),
reverse=True)
computations = [t[:-1] for t in computations]
logger.info("placing computations %s",
[(f, c.name) for f, c, _ in computations])
current_mapping = {} # Type: Dict[str, str]
i = 0
while len(current_mapping) != len(computations):
footprint, computation, candidates = computations[i]
logger.debug(
"Trying to place computation %s with footprint %s",
computation.name,
footprint,
)
# look for cancidiate agents for computation c
# TODO: keep a list of remaining capacities for agents ?
if candidates is None:
candidates = candidate_hosts(
computation,
footprint,
computations,
agentsdef,
communication_load,
current_mapping,
fixed_mapping,
)
computations[i] = footprint, computation, candidates
logger.debug("Candidates for computation %s : %s", computation.name,
candidates)
if not candidates:
if i == 0:
logger.error(
f"Cannot find a distribution, no candidate for computation {computation}\n"
f" current mapping: {current_mapping}")
raise ImpossibleDistributionException(
f"Impossible Distribution, no candidate for {computation}")
# no candidate : backtrack !
i -= 1
logger.info(
"No candidate for %s, backtrack placement "
"of computation %s (was on %s",
computation.name,
computations[i][1].name,
current_mapping[computations[i][1].name],
)
current_mapping.pop(computations[i][1].name)
# FIXME : eliminate selected agent for previous computation
else:
_, selected = candidates.pop()
current_mapping[computation.name] = selected.name
computations[i] = footprint, computation, candidates
logger.debug("Place computation %s on agent %s", computation.name,
selected.name)
i += 1
# Build the distribution for the mapping
agt_mapping = defaultdict(lambda: [])
for c, a in current_mapping.items():
agt_mapping[a].append(c)
for c, (a, _) in fixed_mapping.items():
agt_mapping[a].append(c)
dist = Distribution(agt_mapping)
return dist
def generate(args):
slots, events, resources = generate_problem_definition(
args.slots_count,
args.resources_count,
args.max_resource_value,
args.events_count,
args.max_length_event,
args.max_resources_event,
)
penalty = args.max_resource_value * args.slots_count * args.resources_count
variables, constraints, agents = peav_model(slots, events, resources,
penalty)
domains = {
variable.domain.name: variable.domain
for variable in variables.values()
}
variables = {variable.name: variable for variable in variables.values()}
# agents_defs = {agent.name: agent for agent, _ in agents.values()}
# Generate agents hosting and route costs
agents_defs = {}
if not args.no_agents:
for agent, agt_variables in agents.items():
kw = {}
kw["hosting_costs"] = {v.name: 0 for v in agt_variables}
if args.hosting_default:
kw["default_hosting_cost"] = args.hosting_default
if args.capacity:
kw["capacity"] = args.capacity
if args.routes_default:
kw["default_route"] = args.routes_default
agents_defs[agent] = AgentDef(agent, **kw)
dcop = DCOP(
"MeetingSceduling",
objective="max",
domains=domains,
variables=variables,
constraints=constraints,
agents=agents_defs,
)
if not args.no_agents:
distribution = Distribution({
agent.name: [v.name for v in agents[agent.name]]
for agent in agents_defs.values()
})
if args.output:
output_file = args.output
with open(output_file, encoding="utf-8", mode="w") as fo:
fo.write(dcop_yaml(dcop))
if not args.no_agents:
dist_result = {
"inputs": {
"dist_algo": "peav",
"dcop": output_file,
"graph": "constraints_graph",
"algo": "NA",
},
"distribution": distribution.mapping(),
"cost": None,
}
path, ext = splitext(output_file)
dist_output_file = f"{path}_dist{ext}"
with open(dist_output_file, encoding="utf-8", mode="w") as fo:
fo.write(yaml.dump(dist_result))
else:
print(dcop_yaml(dcop))
if not args.no_agents:
dist_result = {
"inputs": {
"dist_algo": "peav",
"dcop": "NA",
"graph": "constraints_graph",
"algo": "NA",
},
"distribution": distribution.mapping(),
"cost": None,
}
# FIXME proper serialization of the distribution:
print(yaml.dump(dist_result))
def cg_secp_ilp(
cg: ComputationConstraintsHyperGraph,
agents: List[AgentDef],
already_assigned: Distribution,
computation_memory: Callable[[ComputationNode], float],
communication_load: Callable[[ComputationNode, str], float],
timeout=600, # Max 10 min
) -> Distribution:
start_t = time.time()
agents = list(agents)
agents_names = [a.name for a in agents]
# Only keep computations for which we actually need to find an agent.
comps_to_host = [
c for c in cg.node_names() if not already_assigned.has_computation(c)
]
# x_i^k : binary variable indicating if var x_i is hosted on agent a_k.
xs = _build_cs_binvar(comps_to_host, agents_names)
# alpha_ijk : binary variable indicating if x_i and f_j are both on a_k.
alphas = _build_alphaijk_binvars(cg, agents_names)
logger.debug(f"alpha_ijk {alphas}")
# LP problem with objective function (total communication cost).
pb = LpProblem("distribution", LpMinimize)
pb += (
_objective_function(cg, communication_load, alphas, agents_names),
"Communication costs",
)
# Constraints.
# All variable computations must be hosted:
for i in comps_to_host:
pb += (
lpSum([xs[(i, k)] for k in agents_names]) == 1,
"var {} is hosted".format(i),
)
# Each agent must host at least one computation:
# We only need this constraints for agents that do not already host a
# computation:
empty_agents = [
a for a in agents_names if not already_assigned.computations_hosted(a)
]
for k in empty_agents:
pb += (
lpSum([xs[(i, k)] for i in comps_to_host]) >= 1,
"atleastone {}".format(k),
)
# Memory capacity constraint for agents
for a in agents:
# Decrease capacity for already hosted computations
capacity = a.capacity - sum([
secp_computation_memory_in_cg(c, cg, computation_memory)
for c in already_assigned.computations_hosted(a.name)
])
pb += (
lpSum([
secp_computation_memory_in_cg(i, cg, computation_memory) *
xs[(i, a.name)] for i in comps_to_host
]) <= capacity,
"memory {}".format(a.name),
)
# Linearization constraints for alpha_ijk.
for (i, j), k in alphas:
if i in comps_to_host and j in comps_to_host:
pb += alphas[((i, j), k)] <= xs[(i, k)], "lin1 {}{}{}".format(
i, j, k)
pb += alphas[((i, j), k)] <= xs[(j, k)], "lin2 {}{}{}".format(
i, j, k)
pb += (
alphas[((i, j), k)] >= xs[(i, k)] + xs[(j, k)] - 1,
"lin3 {}{}{}".format(i, j, k),
)
elif i in comps_to_host and j not in comps_to_host:
# Var is free, factor is already hosted
if already_assigned.agent_for(j) == k:
pb += alphas[((i, j), k)] == xs[(i, k)]
else:
pb += alphas[((i, j), k)] == 0
elif i not in comps_to_host and j in comps_to_host:
# if i is not in vars_vars_to_host, it means that it's a
# computation that is already hosted (from hints)
if already_assigned.agent_for(i) == k:
pb += alphas[((i, j), k)] == xs[(j, k)]
else:
pb += alphas[((i, j), k)] == 0
else:
# i and j are both alredy hosted
if (already_assigned.agent_for(i) == k
and already_assigned.agent_for(j) == k):
pb += alphas[((i, j), k)] == 1
else:
pb += alphas[((i, j), k)] == 0
# the timeout for the solver must be monierd by the time spent to build the pb:
remaining_time = round(timeout - (time.time() - start_t)) - 2
# Now solve our LP
status = pb.solve(
GLPK_CMD(keepFiles=0,
msg=False,
options=["--pcost", "--tmlim",
str(remaining_time)]))
if status != LpStatusOptimal:
raise ImpossibleDistributionException("No possible optimal"
" distribution ")
else:
logger.debug("GLPK cost : %s", pulp.value(pb.objective))
comp_dist = already_assigned
for k in agents_names:
agt_vars = [
i for i, ka in xs if ka == k and pulp.value(xs[(i, ka)]) == 1
]
comp_dist.host_on_agent(k, agt_vars)
return comp_dist
def test_is_hosted_single_computation(self):
d = Distribution({'a1': ['v1'], 'a2': ['v2']})
self.assertTrue(d.is_hosted('v1'))
self.assertTrue(d.is_hosted('v2'))
self.assertFalse(d.is_hosted('v3'))
def test_host_on_agent_raises_on_already_hosted_comp(self):
d = Distribution({'a1': ['v1', 'v2'], 'a2': ['v3']})
self.assertRaises(ValueError, d.host_on_agent, 'a1', ['v3'])
def generate_iot(args):
print("generate iot ", args.output)
# Constraints and variables with a power-law constraint graph:
variables, constraints, domain = generate_powerlaw_var_constraints(
args.num, args.domain, args.range
)
# Build a dcop and computation graph with no agents, just to be able to
# compute the footprint of computations:
dcop = DCOP(
"graph coloring",
"min",
domains={"d": domain},
variables=variables,
agents={},
constraints=constraints,
)
graph_module = import_module("pydcop.computations_graph.factor_graph")
cg = graph_module.build_computation_graph(dcop)
algo_module = load_algorithm_module("maxsum")
footprints = {c.name: algo_module.computation_memory(c) for c in cg.nodes}
# Generate an agent for each variable computation and assign the
# computation to that agent.
agents = {} # type: Dict[str, AgentDef]
mapping = defaultdict(lambda: []) # type: Dict[str, List[str]]
for comp in cg.nodes:
if isinstance(comp, VariableComputationNode):
a_name = agt_name(comp.name)
agt = AgentDef(
a_name,
capacity=footprints[comp.name] * 100,
default_hosting_cost=10,
hosting_costs=agt_hosting_costs(comp, cg),
default_route=1,
routes=agt_route_costs(comp, cg),
)
logger.debug(
"Create agent %s for computation %s with capacity %s",
agt.name,
comp.name,
agt.capacity,
)
agents[agt.name] = agt
mapping[agt.name].append(comp.name)
# Now, we have created all the agents and distributed all the variables
# let's distribute the factor computations.
msg_load = msg_load_func(cg, algo_module.communication_load)
factor_mapping = distribute_factors(agents, cg, footprints, mapping, msg_load)
for a in mapping:
mapping[a].extend(factor_mapping[a])
dcop = DCOP(
"graph coloring",
"min",
domains={"d": domain},
variables=variables,
agents=agents,
constraints=constraints,
)
distribution = Distribution(mapping)
if args.output:
outputfile = args.output
write_in_file(outputfile, dcop_yaml(dcop))
dist = distribution.mapping()
cost = ilp_compref.distribution_cost(
distribution,
cg,
dcop.agents.values(),
computation_memory=algo_module.computation_memory,
communication_load=algo_module.communication_load,
)
result = {
"inputs": {
"dist_algo": "io_problem",
"dcop": args.output,
"graph": "factor_graph",
"algo": "maxsum",
},
"distribution": dist,
"cost": cost,
}
outputfile = "dist_" + args.output
write_in_file(outputfile, yaml.dump(result))
else:
print(dcop_yaml(dcop))
def distribute(computation_graph: ComputationGraph,
agentsdef: Iterable[AgentDef],
hints: DistributionHints=None,
computation_memory: Callable[[ComputationNode], float]=None,
communication_load: Callable[[ComputationNode, str],
float]=None) \
-> Distribution:
"""
Parameters
----------
computation_graph
agentsdef
hints
computation_memory
communication_load
Returns
-------
"""
computations = sorted([(computation_memory(n), n, None)
for n in computation_graph.nodes],
key=lambda o: (o[0], o[1].name),
reverse=True)
logger.info('placing computations %s',
[(f, c.name) for f, c, _ in computations])
current_mapping = {} # Type: Dict[str, str]
i = 0
while len(current_mapping) != len(computations):
footprint, computation, candidates = computations[i]
logger.debug('Trying to place computation %s with footprint %s',
computation.name, footprint)
# try
# look for agent for computation c
if candidates is None:
candidates = candidate_hosts(computation, footprint, computations,
agentsdef, communication_load,
current_mapping)
computations[i] = footprint, computation, candidates
logger.debug('Candidates for computation %s : %s', computation.name,
candidates)
if not candidates:
if i == 0:
raise ValueError('Impossible Distribution !')
# no candidate : backtrack !
i -= 1
logger.info(
'No candidate for %s, backtrack placement '
'of computation %s (was on %s', computation.name,
computations[i][1].name,
current_mapping[computations[i][1].name])
current_mapping.pop(computations[i][1].name)
# FIXME : eliminate selected agent for previous computation
else:
_, selected = candidates.pop()
current_mapping[computation.name] = selected.name
computations[i] = footprint, computation, candidates
logger.debug('Place computation %s on agent %s', computation.name,
selected.name)
i += 1
# Build the distribution for the mapping
agt_mapping = defaultdict(lambda: [])
for c, a in current_mapping.items():
agt_mapping[a].append(c)
dist = Distribution(agt_mapping)
return dist