def build_algo_def(algo_module, algo_name: str, objective,
cli_params: List[str]):
"""
Build the AlgoDef, which contains the full algorithm specification (
name, objective and parameters)
:param algo_module:
:param algo_name:
:param objective:
:param cli_params:
:return:
"""
# Parameters for the algorithm:
if hasattr(algo_module, 'algo_params'):
params = {}
if cli_params is not None:
logger.info('Using cli params for %s : %s', algo_name, cli_params)
for p in cli_params:
p, v = p.split(':')
params[p] = v
else:
logger.info('Using default parameters for %s', algo_name)
try:
algo_params = algo_module.algo_params(params)
return AlgoDef(algo_name, objective, **algo_params)
except Exception as e:
_error(e)
else:
if cli_params:
_error('Algo {} does not support any parameter'.format(algo_name))
return AlgoDef(algo_name, objective)
def test_algo_def(self):
a = AlgoDef('maxsum', 'min', stability_coef=0.01)
self.assertEqual(a.algo, 'maxsum')
self.assertEqual(a.mode, 'min')
self.assertIn('stability_coef', a.param_names())
self.assertEqual(a.param_value('stability_coef'), 0.01)
def test_from_repr(self):
a = AlgoDef('maxsum', 'min', stability_coef=0.01)
r = simple_repr(a)
a2 = from_repr(r)
self.assertEqual(a, a2)
self.assertEqual(a2.param_value('stability_coef'), 0.01)
def test_simple_repr(self):
a = AlgoDef('maxsum', 'min', stability_coef=0.01)
r = simple_repr(a)
self.assertEqual(r['algo'], 'maxsum')
self.assertEqual(r['mode'], 'min')
self.assertEqual(r['params']['stability_coef'], 0.01)
def test_footprint_on_computation_object():
v1 = Variable('v1', [0, 1, 2, 3, 4])
v2 = Variable('v2', [0, 1, 2, 3, 4])
c1 = relation_from_str('c1', '0 if v1 == v2 else 1', [v1, v2])
n1 = VariableComputationNode(v1, [c1])
n2 = VariableComputationNode(v2, [c1])
comp_def = ComputationDef(n1, AlgoDef('dsa', mode='min'))
c = build_computation(comp_def)
assert c.footprint() == 5
def test_deploy_computation_request(orchestrated_agent):
orchestrated_agent.start()
orchestrated_agent.add_computation = MagicMock()
mgt = orchestrated_agent._mgt_computation
v1 = Variable('v1', [1, 2, 3])
comp_node = VariableComputationNode(v1, [])
comp_def = ComputationDef(comp_node, AlgoDef('dsa'))
mgt.on_message('orchestrator', DeployMessage(comp_def), 0)
# Check the computation is deployed, but not started, on the agent
calls = orchestrated_agent.add_computation.mock_calls
assert len(calls) == 1
_, args, _ = calls[0]
computation = args[0]
assert isinstance(computation, MessagePassingComputation)
assert not computation.is_running
def setup_repair(self, repair_info):
self.logger.info('Setup repair %s', repair_info)
# create computation for the reparation dcop
# The reparation dcop uses a dcop algorithm where computations maps to
# variable (in order to have another dcop distribution problem) and use
# binary variable for each candidate computation.
# This agent will host one variable-computation for each
# binary variable x_i^m indicating if the candidate computation x_i
# is hosted on this agent a_m. Notice that by construction,
# the agent already have a replica for all the candidates x_i.
# The reparation dcop includes several constraints and variables:
# Variables
# * one binary variable for each orphaned computation
# Constraints
# * hosted constraints : one for each candidate computation
# * capacity constraint : one for this agent
# * hosting costs constraint : one for this agent
# * communication constraint
#
# For reparation, we use a dcop algorithm where computations maps to
# variables of the dcop. On this agent, we host the computations
# corresponding to the variables representing the orphaned computation
# that could be hosted on this agent (aka candidate computation).
# Here, we use MGM
own_name = self.name
# `orphaned_binvars` is a map that contains binary variables for
# orphaned computations.
# Notice that it only contains variables for computations
# that this agents knows of, i.e. computations that could be hosted
# here (aka candidate computations) or that depends on computations
# that could be hosted here.
# There is one binary variable x_i^m for each pair (x_i, a_m),
# where x_i is an orphaned computation and a_m is an agent that could
# host x_i (i.e. has a replica of x_i).
orphaned_binvars = {} # type: Dict[Tuple, BinaryVariable]
# One binary variable x_i^m for each candidate computation x_i that
# could be hosted on this agent a_m. Computation for these variables
# will be hosted in this agent. This is a subset of orphaned_binvars.
candidate_binvars = {} # type: Dict[Tuple, BinaryVariable]
# Agent that will host the computation for each binary var.
# it is a dict { bin var name : agent_name }
# agt_hosting_binvar = {} # type: Dict[str, str]
# `hosted_cs` contains hard constraints ensuring that all candidate
# computations are hosted:
hosted_cs = {} # type: Dict[str, Constraint]
for candidate_comp, candidate_info in repair_info.items():
try:
# This computation is not hosted any more, if we had it in
# discovery, forget about it but do not publish this
# information, this agent is not responsible for updatings
# other's discovery services.
self.discovery.unregister_computation(candidate_comp,
publish=False)
except UnknownComputation:
pass
agts, _, neighbors = candidate_info
# One binary variable for each candidate agent for computation
# candidate_comp:
v_binvar = create_binary_variables(
'B', ([candidate_comp], candidate_info[0]))
orphaned_binvars.update(v_binvar)
# the variable representing if the computation will be hosted on
# this agent:
candidate_binvars[(candidate_comp, own_name)] = \
v_binvar[(candidate_comp, own_name)]
# the 'hosted' hard constraint for this candidate variable:
hosted_cs[candidate_comp] =\
create_computation_hosted_constraint(candidate_comp, v_binvar)
self.logger.debug('Hosted hard constraint for computation %s : %r',
candidate_comp, hosted_cs[candidate_comp])
# One binary variable for each pair (x_j, a_n) where x_j is an
# orphaned neighbors of candidate_comp and a_n is an agent that
# could host a_n:
for neighbor in neighbors:
v_binvar = create_binary_variables(
'B', ([neighbor], neighbors[neighbor]))
orphaned_binvars.update(v_binvar)
self.logger.debug('Binary variable for reparation %s ',
orphaned_binvars)
# Agent that will host the computation for each binary var.
# it is a dict { bin var name : agent_name }
agt_hosting_binvar = {
v.name: a
for (_, a), v in orphaned_binvars.items()
}
self.logger.debug(
'Agents hosting the computations for these binary '
'variables : %s ', agt_hosting_binvar)
# The capacity (hard) constraint for this agent. This ensures that the
# capacity of the current agent will not be overflown by hosting too
# many candidate computations. This constraints depends on the binary
# variables for the candidate computations.
remaining_capacity = self.agent_def.capacity - \
sum(c.footprint() for c in self.computations())
self.logger.debug('Remaining capacity on agent %s : %s', self.name,
remaining_capacity)
def footprint_func(c_name: str):
# We have a replica for these computation, we known its footprint.
return self.replication_comp.hosted_replicas[c_name][1]
capacity_c = create_agent_capacity_constraint(own_name,
remaining_capacity,
footprint_func,
candidate_binvars)
self.logger.debug('Capacity constraint for agt %s : %r', self.name,
capacity_c)
# Hosting costs constraint for this agent. This soft constraint is
# used to minimize the hosting costs on this agent ; it depends on
# the binary variables for the candidate computations.
hosting_c = create_agent_hosting_constraint(
own_name, self.agent_def.hosting_cost, candidate_binvars)
self.logger.debug('Hosting cost constraint for agt %s : %r', self.name,
hosting_c)
# The communication constraint. This soft constraints is used to
# minimize the communication cost on this agent. As communication
# cost depends on where computation on both side of an edge are
# hosted, it also depends on the binary variables for orphaned
# computations that could not be hosted here.
def comm_func(candidate_comp: str, neighbor_comp: str, agt: str):
# returns the communication cost between the computation
# candidate_name hosted on the current agent and it's neighbor
# computation neigh_comp hosted on agt.
route_cost = self.agent_def.route(agt)
comp_def = self.replication_comp.replicas[candidate_comp]
algo = comp_def.algo.algo
algo_module = import_module('pydcop.algorithms.{}'.format(algo))
communication_load = algo_module.communication_load
msg_load = 0
for l in comp_def.node.neighbors:
if l == neighbor_comp:
msg_load += communication_load(comp_def.node,
neighbor_comp)
com_load = msg_load * route_cost
return com_load
# Now that we have the variables and constraints, we can create
# computation instances for each of the variable this agent is
# responsible for, i.e. the binary variables x_i^m that correspond to
# the candidate variable x_i (and a_m is the current agent)
self._repair_computations.clear()
algo_def = AlgoDef(repair_algo.algo_name(),
cycle_stop=10,
threshold=0.2)
for (comp, agt), candidate_var in candidate_binvars.items():
self.logger.debug(
'Building computation for binary variable %s ('
'variable %s on %s)', candidate_var, comp, agt)
comm_c = create_agent_comp_comm_constraint(agt, comp,
repair_info[comp],
comm_func,
orphaned_binvars)
self.logger.debug(
'Communication constraint for computation %s '
'on agt %s : %r', comp, self.name, comm_c)
constraints = [comm_c, hosting_c, capacity_c, hosted_cs[comp]]
# constraints.extend(hosted_cs.values())
self.logger.debug('Got %s Constraints for var %s : %s ',
len(constraints), candidate_var, constraints)
node = chg.VariableComputationNode(candidate_var, constraints)
comp_def = ComputationDef(node, algo_def)
computation = repair_algo.build_computation(comp_def)
self.logger.debug('Computation for %s : %r ', candidate_var,
computation)
# add the computation on this agents and register the neighbors
self.add_computation(computation)
self._repair_computations[computation.name] = \
RepairComputationRegistration(computation, 'ready', comp)
for neighbor_comp in node.neighbors:
neighbor_agt = agt_hosting_binvar[neighbor_comp]
try:
self.discovery.register_computation(neighbor_comp,
neighbor_agt,
publish=False)
except UnknownAgent:
# If we don't know this agent yet, we must perform a lookup
# and only register the computation once found.
# Note the use of partial, to force the capture of
# neighbor_comp.
def _agt_lookup_done(comp, evt, evt_agt, _):
if evt == 'agent_added':
self.discovery.register_computation(comp,
evt_agt,
publish=False)
self.discovery.subscribe_agent(neighbor_agt,
partial(
_agt_lookup_done,
neighbor_comp),
one_shot=True)
self.logger.info(
'Repair setup done one %s, %s computations created, '
'inform orchestrator', self.name, len(candidate_binvars))
return candidate_binvars
def solve(dcop: DCOP,
algo_def: Union[str, AlgoDef],
distribution: Union[str, Distribution],
graph: Union[str, ComputationGraph] = None,
timeout=5):
"""Solve a dcop in a single process.
This is simply a convenience method that hides all the complexity of the
orchestrator, agents creation, etc.
Parameters
----------
dcop : DCOP
a DCOP object
algo_def: string or AlgoDef object
The algorithm that should be used to solve the DCOP. If `algo_def` is
a string, it is interpreted as an algorithm module in
the package `pydcop.algorithms`, and this algorithm is used with it's
default parameters. If `algo_def` is not a string, it must be an
AlgoDef object.
distribution: either a Distribution object or a string
If `distribution` is a string, it is interpreted as the name of a module
in the package pydcop.distribution. This module is the loaded and used
to generate a distribution of the DCOP on the agents. If `distribution`
is not a string, it must be a Distribution object that maps
computations to agents.
graph: either a string a ComputationGraph object
If `graph` is a string, it is interpreted as the name of module in
the `pydcop.computations_graph` package ; the module is loaded and
used to build computation graph for the dcop. If `graph` is not a
string, it must be a ComputationGraph object for the given DCOP.
Ths parameter is optional, if it is not given it is deduced from the
algorithm.
timeout:float
in seconds
Returns
-------
the result
Examples
--------
assignment = solve(dcop, 'maxsum', 'adhoc', timeout=3)
"""
if isinstance(algo_def, str):
algo_def = AlgoDef(algo_def)
algo_module = import_module('pydcop.algorithms.' + algo_def.algo)
if graph is None:
graph_module = import_module('pydcop.computations_graph.{}'.format(
algo_module.GRAPH_TYPE))
graph = graph_module.build_computation_graph(dcop)
elif isinstance(graph, str):
graph_module = import_module('pydcop.computations_graph.' + graph)
graph = graph_module.build_computation_graph(dcop)
if isinstance(distribution, str):
distrib_module = import_module('pydcop.distribution.' + distribution)
distribution = distrib_module.distribute(
graph,
dcop.agents.values(),
computation_memory=algo_module.computation_memory,
communication_load=algo_module.communication_load)
orchestrator = run_local_thread_dcop(algo_def, graph, distribution, dcop,
INFINITY)
try:
print('Deploy')
orchestrator.deploy_computations()
print('start Running')
orchestrator.run(timeout=timeout)
print('Running, wait ready')
orchestrator.wait_ready()
print('Done')
return orchestrator.end_metrics()['assignment']
except Exception:
orchestrator.stop_agents(5)
orchestrator.stop()
finally:
orchestrator.stop_agents(5)
orchestrator.stop()