def distribute(
computation_graph: ComputationsFactorGraph,
agentsdef: Iterable[AgentDef],
hints=None,
computation_memory: Callable[[ComputationNode], float] = None,
communication_load: Callable[[ComputationNode, str], float] = None,
timeout=600, # Max 10 min
) -> Distribution:
if computation_memory is None or communication_load is None:
raise ImpossibleDistributionException(
"oilp_secp_fgdp distribution requires "
"computation_memory and link_communication functions"
)
mapping = defaultdict(lambda: [])
agents_capa = {a.name: a.capacity for a in agentsdef}
variable_computations, factor_computations = [], []
for comp in computation_graph.nodes:
if isinstance(comp, VariableComputationNode):
variable_computations.append(comp.name)
elif isinstance(comp, FactorComputationNode):
factor_computations.append(comp.name)
else:
raise ImpossibleDistributionException(
f"Error: {comp} is neither a factor nor a variable computation"
)
# actuators variables and cost factor on the corresponding agent:
for variable in variable_computations[:]:
for agent in agentsdef:
if agent.hosting_cost(variable) == 0:
# Found an actuator variable, host it on the agent
mapping[agent.name].append(variable)
variable_computations.remove(variable)
agents_capa[agent.name] -= computation_memory(
computation_graph.computation(variable)
)
# search for the cost factor, if any, and host it on the same agent.
for factor in factor_computations[:]:
if f"c_{variable}" == factor:
mapping[agent.name].append(factor)
factor_computations.remove(factor)
agents_capa[agent.name] -= computation_memory(
computation_graph.computation(factor)
)
if agents_capa[agent.name] < 0:
raise ImpossibleDistributionException(
f"Not enough capacity on {agent} to hosts actuator {variable}: {agents_capa[agent.name]}"
)
break
logger.info(f"Actuator variables - agents: {dict(mapping)}")
logger.info(f"Remaining capacity: {dict(agents_capa)}")
return fg_secp_ilp(
computation_graph,
agentsdef,
Distribution(mapping),
computation_memory,
communication_load,
)
def test_comm(self):
f1 = relation_from_str('f1', 'v1 * 0.5 + v2 + v3', [v1, v2, v3])
cv1 = VariableComputationNode(v1, ['f1'])
cv2 = VariableComputationNode(v2, ['f1'])
cv3 = VariableComputationNode(v3, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1, cv2, cv3], [cf1])
a1 = AgentDef("a1",
capacity=200,
default_hosting_cost=1,
hosting_costs={
"v1": 0,
"v2": 0
})
a2 = AgentDef("a2", capacity=200, default_hosting_cost=1)
a1.capacity = 1000
agent_mapping = distribute(cg, [a1, a2],
hints=None,
computation_memory=ms.computation_memory,
communication_load=ms.communication_load)
# As there is enough capacity on a1, factor f1 must go there (where
# most of its variable are already hosted) while v3 must go on a2 to
# make sure that all agents are used
self.assertEqual(agent_mapping.agent_for('f1'), 'a1')
self.assertEqual(agent_mapping.agent_for('v3'), 'a2')
def test_respect_must_host_all_computation_fixed(self):
f1 = relation_from_str('f1', 'v1 * 0.5 + v2', [v1, v2])
cv1 = VariableComputationNode(v1, ['f1'])
cv2 = VariableComputationNode(v2, [])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1, cv2], [cf1])
a1 = AgentDef("a1",
capacity=200,
default_hosting_cost=1,
hosting_costs={
"f1": 0,
"v1": 0
})
a2 = AgentDef("a2",
capacity=200,
default_hosting_cost=1,
hosting_costs={"v2": 0})
agent_mapping = distribute(cg, [a1, a2],
hints=None,
computation_memory=ms.computation_memory,
communication_load=ms.communication_load)
self.assertEqual(agent_mapping.agent_for('f1'), 'a1')
self.assertEqual(agent_mapping.agent_for('v1'), 'a1')
self.assertEqual(agent_mapping.agent_for('v2'), 'a2')
def test_host_on_highest_dependent_agent(self):
d1 = VariableDomain('d1', '', [1, 2, 3, 5])
v1 = Variable('v1', d1)
v2 = Variable('v2', d1)
v3 = Variable('v3', d1)
f1 = relation_from_str('f1', 'v1 + v2', [v1, v2])
f2 = relation_from_str('f2', 'v1 - v2 + v3', [v1, v2, v3])
cv1 = VariableComputationNode(v1, ['f1', 'f2'])
cv2 = VariableComputationNode(v2, ['f1', 'f2'])
cv3 = VariableComputationNode(v3, ['f2'])
cf1 = FactorComputationNode(f1)
cf2 = FactorComputationNode(f2)
cg = ComputationsFactorGraph([cv1, cv2, cv3], [cf1, cf2])
hints = DistributionHints(must_host={'a1': ['v1'], 'a2': ['v2', 'v3']})
# we must set the capacity to make sure that a2 cannot take f1
agents = [AgentDef('a{}'.format(i), capacity=41) for i in range(1, 11)]
agent_mapping = distribute(cg,
agents,
hints,
computation_memory=lambda x: 10)
print(agent_mapping)
self.assertEqual(agent_mapping.agent_for('f1'), 'a1')
self.assertEqual(agent_mapping.agent_for('f2'), 'a2')
self.assertTrue(is_all_hosted(cg, agent_mapping))
def test_respect_must_host_all_computation_invalid(self):
f1 = relation_from_str('f1', 'v1 * 0.5', [v1])
cv1 = VariableComputationNode(v1, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1], [cf1])
a1 = AgentDef("a1",
capacity=200,
default_hosting_cost=1,
hosting_costs={
"f1": 0,
"v1": 0
})
a2 = AgentDef("a2", capacity=200, default_hosting_cost=1)
# These hints lead to an impossible distribution, as ilp-fgdp requires
# each agent to host at least one computation. Here Both
# computations are hosted on a1 and there is no computation
# available for a2 !
self.assertRaises(ImpossibleDistributionException,
distribute,
cg, [a1, a2],
hints=None,
computation_memory=ms.computation_memory,
communication_load=ms.communication_load)
def test_comm_not_enough_place(self):
f1 = relation_from_str('f1', 'v1 * 0.5 + v2 + v3', [v1, v2, v3])
cv1 = VariableComputationNode(v1, ['f1'])
cv2 = VariableComputationNode(v2, ['f1'])
cv3 = VariableComputationNode(v3, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1, cv2, cv3], [cf1])
a1 = AgentDef("a1",
capacity=200,
default_hosting_cost=1,
hosting_costs={
"v1": 0,
"v2": 0
})
a2 = AgentDef("a2", capacity=200, default_hosting_cost=1)
a1.capacity = 15
agent_mapping = distribute(cg, [a1, a2],
hints=None,
computation_memory=ms.computation_memory,
communication_load=ms.communication_load)
# As there is enough not capacity on a1, factor f1 and variable v3
# must go on a2
self.assertEqual(agent_mapping.agent_for('f1'), 'a2')
self.assertEqual(agent_mapping.agent_for('v3'), 'a2')
def test_raises_if_methods_not_given(self):
f1 = relation_from_str('f1', 'v1 * 0.5', [v1])
cv1 = VariableComputationNode(v1, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1], [cf1])
self.assertRaises(ImpossibleDistributionException, distribute, cg,
[Agent('a1'), Agent('a2')], hints=None)
def test_create_ok(self):
d1 = Domain('d1', '', [1, 2, 3, 5])
v1 = Variable('v1', d1)
f1 = constraint_from_str('f1', 'v1 * 0.5', [v1])
cv1 = VariableComputationNode(v1, [f1])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1], [cf1])
def test_var_fac_nolink(self):
f1 = relation_from_str('f1', '0.5', [])
cv1 = VariableComputationNode(v1, [])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1], [cf1])
self.assertEqual(
_computation_memory_in_cg('v1', cg, computation_memory), 0)
self.assertEqual(
_computation_memory_in_cg('f1', cg, computation_memory), 0)
def test_fac_2var(self):
f1 = relation_from_str('f1', 'v1 * 0.5+v2', [v1, v2])
cv1 = VariableComputationNode(v1, ['f1'])
cv2 = VariableComputationNode(v2, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1, cv2], [cf1])
# size of the domain of v1 + size domain v2
self.assertEqual(
_computation_memory_in_cg('f1', cg, computation_memory), 8)
def secp_dist_objective_function(cg: ComputationsFactorGraph,
communication_load, alphas, agents_names):
# The objective function is the negated sum of the communication cost on
# the links in the factor graph.
return lpSum([
-communication_load(cg.computation(link.variable_node),
link.factor_node) *
alphas[((link.variable_node, link.factor_node), k)]
for link in cg.links for k in agents_names
])
def test_raise_when_not_enough_agents(self):
d1 = VariableDomain('d1', '', [1, 2, 3, 5])
v1 = Variable('v1', d1)
f1 = relation_from_str('f1', 'v1 * 0.5', [v1])
cv1 = VariableComputationNode(v1, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1], [cf1])
self.assertRaises(ImpossibleDistributionException, distribute, cg,
[AgentDef('a1')])
def test_no_hints(self):
d1 = VariableDomain('d1', '', [1, 2, 3, 5])
v1 = Variable('v1', d1)
f1 = relation_from_str('f1', 'v1 * 0.5', [v1])
cv1 = VariableComputationNode(v1, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1], [cf1])
agents = [AgentDef('a1', capacity=100), AgentDef('a2', capacity=100)]
agent_mapping = distribute(cg, agents, hints=None,
computation_memory=lambda x: 10)
self.assertTrue(agent_mapping.is_hosted(['v1', 'f1']))
def test_var_fac_link(self):
f1 = relation_from_str('f1', 'v1 * 0.5', [v1])
cv1 = VariableComputationNode(v1, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1], [cf1])
# size of the domain of v1
self.assertEqual(
_computation_memory_in_cg('v1', cg, computation_memory),
4 * VARIABLE_UNIT_SIZE)
self.assertEqual(
_computation_memory_in_cg('f1', cg, computation_memory), 4)
def test_obj_function(self):
f1 = relation_from_str('f1', 'v1 * 0.5', [v1])
cv1 = VariableComputationNode(v1, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1], [cf1])
agents_names = ['a1', 'a2']
alphas = _build_alphaijk_binvars(cg, agents_names)
obj = _objective_function(cg, communication_load, alphas, agents_names)
# In that case, the objective function must depend on two variables:
self.assertEqual(len(obj.sorted_keys()), 2)
def test_build_alphaijk_one_var_one_fac(self):
f1 = relation_from_str('f1', 'v1 * 0.5', [v1])
cv1 = VariableComputationNode(v1, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1], [cf1])
agents_names = ['a1', 'a2']
alphas = _build_alphaijk_binvars(cg, agents_names)
self.assertEqual(len(alphas), 2)
self.assertIn((('v1', 'f1'), 'a1'), alphas)
self.assertIn((('v1', 'f1'), 'a2'), alphas)
print(alphas)
def test_respect_must_host_for_var(self):
f1 = relation_from_str('f1', 'v1 * 0.5', [v1])
cv1 = VariableComputationNode(v1, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1], [cf1])
hints = DistributionHints(must_host={'a1': ['v1']})
agent_mapping = distribute(cg, [a1, a2],
hints=hints,
computation_memory=ms.computation_memory,
communication_load=ms.communication_load)
self.assertEqual(agent_mapping.agent_for('v1'), 'a1')
def test_must_host_one(self):
d1 = VariableDomain('d1', '', [1, 2, 3, 5])
v1 = Variable('v1', d1)
f1 = relation_from_str('f1', 'v1 * 0.5', [v1])
cv1 = VariableComputationNode(v1, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1], [cf1])
hints = DistributionHints({'a1': ['v1']}, None)
agents = [AgentDef('a1', capacity=100), AgentDef('a2', capacity=100)]
agent_mapping = distribute(cg, agents, hints,
computation_memory=lambda x: 10)
self.assertIn('v1', agent_mapping.computations_hosted('a1'))
self.assertTrue(is_all_hosted(cg, agent_mapping))
def test_respect_must_host_all_computation_invalid(self):
f1 = relation_from_str('f1', 'v1 * 0.5', [v1])
cv1 = VariableComputationNode(v1, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1], [cf1])
hints = DistributionHints(must_host={'a1': ['f1', 'v1']})
# These hints lead to an impossible distribution, as ilp-fgdp requires
# each agent to host at least one computation. Here Both
# computations are hosted on a1 and there is no computation
# available for a2 !
self.assertRaises(ImpossibleDistributionException, distribute,
cg, [a1, a2], hints=hints,
computation_memory=ms.computation_memory,
communication_load=ms.communication_load)
def test_simple_fg(self):
d1 = VariableDomain('d1', '', [1, 2, 3, 5])
v1 = Variable('v1', d1)
f1 = relation_from_str('f1', 'v1 * 0.5', [v1])
cv1 = VariableComputationNode(v1, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1], [cf1])
agents= [AgentDef('a1'), AgentDef('a2')]
distribution = distribute(cg, agents)
self.assertEqual(len(distribution.agents), 2)
self.assertEqual(len(distribution.computations), 2)
self.assertEqual(len(distribution.computations_hosted('a1')), 1)
self.assertEqual(len(distribution.computations_hosted('a2')), 1)
self.assertNotEqual(distribution.computations_hosted('a2'),
distribution.computations_hosted('a1'))
def test_comm_not_enough_place(self):
f1 = relation_from_str('f1', 'v1 * 0.5 + v2 + v3', [v1, v2, v3])
cv1 = VariableComputationNode(v1, ['f1'])
cv2 = VariableComputationNode(v2, ['f1'])
cv3 = VariableComputationNode(v3, ['f1'])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1, cv2, cv3], [cf1])
hints = DistributionHints(must_host={'a1': ['v1', 'v2']})
a1.capacity = 10
agent_mapping = distribute(cg, [a1, a2],
hints=hints,
computation_memory=ms.computation_memory,
communication_load=ms.communication_load)
# As there is enough not capacity on a1, factor f1 and variable v3
# must go on a2
self.assertEqual(agent_mapping.agent_for('f1'), 'a2')
self.assertEqual(agent_mapping.agent_for('v3'), 'a2')
def setUp(self):
d1 = VariableDomain('d1', '', [1, 2, 3, 5])
# An secp made of 2 lights, one model and two rule
v1 = Variable('v1', d1)
v2 = Variable('v2', d1)
v3 = Variable('v3', d1)
m1 = Variable('m1', d1)
mf1 = relation_from_str('mf1', 'v1 + v2 == m1', [v1, v2, m1])
r1 = relation_from_str('r1', 'm1 - v2', [m1, v2])
r2 = relation_from_str('r2', 'v3', [v3])
cv1 = VariableComputationNode(v1, ['mf1'])
cv2 = VariableComputationNode(v2, ['mf1', 'r1'])
cv3 = VariableComputationNode(v3, ['r2'])
cm1 = VariableComputationNode(m1, ['mf1', 'r1'])
cmf1 = FactorComputationNode(mf1)
cr1 = FactorComputationNode(r1)
cr2 = FactorComputationNode(r2)
self.cg = ComputationsFactorGraph([cv1, cv2, cv3, cm1],
[cmf1, cr1, cr2])
def distribution_cost(
distribution: Distribution,
computation_graph: ComputationsFactorGraph,
agentsdef: Iterable[AgentDef],
computation_memory: Callable[[ComputationNode], float],
communication_load: Callable[[ComputationNode, str], float],
) -> float:
"""
Compute the cost of the distribution.
Only takes communication costs into account (no hosting nor route costs).
Parameters
----------
distribution
computation_graph
agentsdef
computation_memory
communication_load
Returns
-------
"""
comm = 0
agt_names = [a.name for a in agentsdef]
for l in computation_graph.links:
# As we support hypergraph, we may have more than 2 ends to a link
for c1, c2 in combinations(l.nodes, 2):
if distribution.agent_for(c1) != distribution.agent_for(c2):
edge_cost = communication_load(
computation_graph.computation(c1), c2)
logger.debug(f"edge cost between {c1} and {c2} : {edge_cost}")
comm += edge_cost
else:
logger.debug(
f"On same agent, no edge cost between {c1} and {c2}")
# This distribution model only takes communication cost into account.
# cost = RATIO_HOST_COMM * comm + (1-RATIO_HOST_COMM) * hosting
return comm, comm, 0
def test_host_with(self):
d1 = VariableDomain('d1', '', [1, 2, 3, 5])
v1 = Variable('v1', d1)
v2 = Variable('v2', d1)
f1 = relation_from_str('f1', 'v1 * 0.5', [v1])
cv1 = VariableComputationNode(v1, ['f1'])
cv2 = VariableComputationNode(v2, [])
cf1 = FactorComputationNode(f1)
cg = ComputationsFactorGraph([cv1, cv2], [cf1])
hints = DistributionHints(None, {'v1': ['f1']})
agents = [AgentDef('a{}'.format(i), capacity=100)
for i in range(1, 11)]
agent_mapping = distribute(cg, agents, hints,
computation_memory=lambda x: 10)
self.assertEqual(agent_mapping.agent_for('v1'),
agent_mapping.agent_for('f1'))
self.assertTrue(is_all_hosted(cg, agent_mapping))
def secp_computation_memory_in_cg(computation_name: str,
cg: ComputationsFactorGraph,
computation_memory):
computation = cg.computation(computation_name)
l = computation_memory(computation)
return l
def distribute(
computation_graph: ComputationsFactorGraph,
agentsdef: Iterable[AgentDef],
hints: DistributionHints = None,
computation_memory: Callable[[ComputationNode], float] = None,
communication_load: Callable[[ComputationNode, str], float] = None,
timeout=None, # not used
) -> Distribution:
if computation_memory is None:
raise ImpossibleDistributionException("adhoc distribution requires "
"computation_memory functions")
# as we're dealing with a secp modelled as a factor graph, we have computations for
# actuator and physical model variables, rules and physical model factors.
mapping = defaultdict(lambda: [])
agents_capa = {a.name: a.capacity for a in agentsdef}
variable_computations = []
factor_computations = []
for comp in computation_graph.nodes:
if isinstance(comp, VariableComputationNode):
variable_computations.append(comp.name)
elif isinstance(comp, FactorComputationNode):
factor_computations.append(comp.name)
else:
raise ImpossibleDistributionException(
f"Error: {comp} is neither a factor nor a variable computation"
)
# First, put each actuator variable and cost factor on its agent
for variable in variable_computations[:]:
for agent in agentsdef:
if agent.hosting_cost(variable) == 0:
# Found an actuator variable, host it on the agent
mapping[agent.name].append(variable)
variable_computations.remove(variable)
agents_capa[agent.name] -= computation_memory(
computation_graph.computation(variable))
# search for the cost factor, if any, and host it on the same agent.
for factor in factor_computations[:]:
if f"c_{variable}" == factor:
mapping[agent.name].append(factor)
factor_computations.remove(factor)
agents_capa[agent.name] -= computation_memory(
computation_graph.computation(factor))
if agents_capa[agent.name] < 0:
raise ImpossibleDistributionException(
f"Not enough capacity on {agent} to hosts actuator {variable}: {agents_capa[agent.name]}"
)
break
logger.info(f"Actuator computations - agents: {dict(mapping)}")
logger.info(f"Remaining capacity: {dict(agents_capa)}")
# now find computations for physical models and variables variables.
# * all remaining variables are model variables
# * physical model factor computation names contain the name of the variable
model_variables = variable_computations
models = []
for model_var in model_variables:
for fact in factor_computations:
if f"c_{model_var}" == fact:
models.append((model_var, fact))
factor_computations.remove(fact)
# All remaining factor ar rule factors
rule_factors = factor_computations
logger.debug(f"Physical models: {models}")
logger.debug(f"Rules: {rule_factors}")
# Now place models
for model_var, model_fac in models:
footprint = computation_memory(
computation_graph.computation(model_fac)) + computation_memory(
computation_graph.computation(model_var))
neighbors = computation_graph.neighbors(model_fac)
candidates = find_candidates(agents_capa, model_fac, footprint,
mapping, neighbors)
# Host the model on the first agent and decrease its remaining capacity
selected = candidates[0][2]
mapping[selected].append(model_var)
mapping[selected].append(model_fac)
agents_capa[selected] -= footprint
logger.debug(f"All models hosted: {dict(mapping)}")
logger.debug(f"Remaining capacity: {agents_capa}")
# And rules at last:
for rule_fac in rule_factors:
footprint = computation_memory(computation_graph.computation(rule_fac))
neighbors = computation_graph.neighbors(rule_fac)
candidates = find_candidates(agents_capa, rule_fac, footprint, mapping,
neighbors)
# Host the computation on the first agent and decrease its remaining capacity
selected = candidates[0][2]
mapping[selected].append(rule_fac)
agents_capa[selected] -= footprint
return Distribution({a: list(mapping[a]) for a in mapping})
