def build_algo_def(algo_module, algo_name: str, objective, cli_params: List[str]): """ Build the AlgoDef, which contains the full algorithm specification ( name, objective and parameters) :param algo_module: :param algo_name: :param objective: :param cli_params: :return: """ # Parameters for the algorithm: if hasattr(algo_module, 'algo_params'): params = {} if cli_params is not None: logger.info('Using cli params for %s : %s', algo_name, cli_params) for p in cli_params: p, v = p.split(':') params[p] = v else: logger.info('Using default parameters for %s', algo_name) try: algo_params = algo_module.algo_params(params) return AlgoDef(algo_name, objective, **algo_params) except Exception as e: _error(e) else: if cli_params: _error('Algo {} does not support any parameter'.format(algo_name)) return AlgoDef(algo_name, objective)
def test_algo_def(self): a = AlgoDef('maxsum', 'min', stability_coef=0.01) self.assertEqual(a.algo, 'maxsum') self.assertEqual(a.mode, 'min') self.assertIn('stability_coef', a.param_names()) self.assertEqual(a.param_value('stability_coef'), 0.01)
def test_from_repr(self): a = AlgoDef('maxsum', 'min', stability_coef=0.01) r = simple_repr(a) a2 = from_repr(r) self.assertEqual(a, a2) self.assertEqual(a2.param_value('stability_coef'), 0.01)
def test_simple_repr(self): a = AlgoDef('maxsum', 'min', stability_coef=0.01) r = simple_repr(a) self.assertEqual(r['algo'], 'maxsum') self.assertEqual(r['mode'], 'min') self.assertEqual(r['params']['stability_coef'], 0.01)
def test_footprint_on_computation_object(): v1 = Variable('v1', [0, 1, 2, 3, 4]) v2 = Variable('v2', [0, 1, 2, 3, 4]) c1 = relation_from_str('c1', '0 if v1 == v2 else 1', [v1, v2]) n1 = VariableComputationNode(v1, [c1]) n2 = VariableComputationNode(v2, [c1]) comp_def = ComputationDef(n1, AlgoDef('dsa', mode='min')) c = build_computation(comp_def) assert c.footprint() == 5
def test_deploy_computation_request(orchestrated_agent): orchestrated_agent.start() orchestrated_agent.add_computation = MagicMock() mgt = orchestrated_agent._mgt_computation v1 = Variable('v1', [1, 2, 3]) comp_node = VariableComputationNode(v1, []) comp_def = ComputationDef(comp_node, AlgoDef('dsa')) mgt.on_message('orchestrator', DeployMessage(comp_def), 0) # Check the computation is deployed, but not started, on the agent calls = orchestrated_agent.add_computation.mock_calls assert len(calls) == 1 _, args, _ = calls[0] computation = args[0] assert isinstance(computation, MessagePassingComputation) assert not computation.is_running
def setup_repair(self, repair_info): self.logger.info('Setup repair %s', repair_info) # create computation for the reparation dcop # The reparation dcop uses a dcop algorithm where computations maps to # variable (in order to have another dcop distribution problem) and use # binary variable for each candidate computation. # This agent will host one variable-computation for each # binary variable x_i^m indicating if the candidate computation x_i # is hosted on this agent a_m. Notice that by construction, # the agent already have a replica for all the candidates x_i. # The reparation dcop includes several constraints and variables: # Variables # * one binary variable for each orphaned computation # Constraints # * hosted constraints : one for each candidate computation # * capacity constraint : one for this agent # * hosting costs constraint : one for this agent # * communication constraint # # For reparation, we use a dcop algorithm where computations maps to # variables of the dcop. On this agent, we host the computations # corresponding to the variables representing the orphaned computation # that could be hosted on this agent (aka candidate computation). # Here, we use MGM own_name = self.name # `orphaned_binvars` is a map that contains binary variables for # orphaned computations. # Notice that it only contains variables for computations # that this agents knows of, i.e. computations that could be hosted # here (aka candidate computations) or that depends on computations # that could be hosted here. # There is one binary variable x_i^m for each pair (x_i, a_m), # where x_i is an orphaned computation and a_m is an agent that could # host x_i (i.e. has a replica of x_i). orphaned_binvars = {} # type: Dict[Tuple, BinaryVariable] # One binary variable x_i^m for each candidate computation x_i that # could be hosted on this agent a_m. Computation for these variables # will be hosted in this agent. This is a subset of orphaned_binvars. candidate_binvars = {} # type: Dict[Tuple, BinaryVariable] # Agent that will host the computation for each binary var. # it is a dict { bin var name : agent_name } # agt_hosting_binvar = {} # type: Dict[str, str] # `hosted_cs` contains hard constraints ensuring that all candidate # computations are hosted: hosted_cs = {} # type: Dict[str, Constraint] for candidate_comp, candidate_info in repair_info.items(): try: # This computation is not hosted any more, if we had it in # discovery, forget about it but do not publish this # information, this agent is not responsible for updatings # other's discovery services. self.discovery.unregister_computation(candidate_comp, publish=False) except UnknownComputation: pass agts, _, neighbors = candidate_info # One binary variable for each candidate agent for computation # candidate_comp: v_binvar = create_binary_variables( 'B', ([candidate_comp], candidate_info[0])) orphaned_binvars.update(v_binvar) # the variable representing if the computation will be hosted on # this agent: candidate_binvars[(candidate_comp, own_name)] = \ v_binvar[(candidate_comp, own_name)] # the 'hosted' hard constraint for this candidate variable: hosted_cs[candidate_comp] =\ create_computation_hosted_constraint(candidate_comp, v_binvar) self.logger.debug('Hosted hard constraint for computation %s : %r', candidate_comp, hosted_cs[candidate_comp]) # One binary variable for each pair (x_j, a_n) where x_j is an # orphaned neighbors of candidate_comp and a_n is an agent that # could host a_n: for neighbor in neighbors: v_binvar = create_binary_variables( 'B', ([neighbor], neighbors[neighbor])) orphaned_binvars.update(v_binvar) self.logger.debug('Binary variable for reparation %s ', orphaned_binvars) # Agent that will host the computation for each binary var. # it is a dict { bin var name : agent_name } agt_hosting_binvar = { v.name: a for (_, a), v in orphaned_binvars.items() } self.logger.debug( 'Agents hosting the computations for these binary ' 'variables : %s ', agt_hosting_binvar) # The capacity (hard) constraint for this agent. This ensures that the # capacity of the current agent will not be overflown by hosting too # many candidate computations. This constraints depends on the binary # variables for the candidate computations. remaining_capacity = self.agent_def.capacity - \ sum(c.footprint() for c in self.computations()) self.logger.debug('Remaining capacity on agent %s : %s', self.name, remaining_capacity) def footprint_func(c_name: str): # We have a replica for these computation, we known its footprint. return self.replication_comp.hosted_replicas[c_name][1] capacity_c = create_agent_capacity_constraint(own_name, remaining_capacity, footprint_func, candidate_binvars) self.logger.debug('Capacity constraint for agt %s : %r', self.name, capacity_c) # Hosting costs constraint for this agent. This soft constraint is # used to minimize the hosting costs on this agent ; it depends on # the binary variables for the candidate computations. hosting_c = create_agent_hosting_constraint( own_name, self.agent_def.hosting_cost, candidate_binvars) self.logger.debug('Hosting cost constraint for agt %s : %r', self.name, hosting_c) # The communication constraint. This soft constraints is used to # minimize the communication cost on this agent. As communication # cost depends on where computation on both side of an edge are # hosted, it also depends on the binary variables for orphaned # computations that could not be hosted here. def comm_func(candidate_comp: str, neighbor_comp: str, agt: str): # returns the communication cost between the computation # candidate_name hosted on the current agent and it's neighbor # computation neigh_comp hosted on agt. route_cost = self.agent_def.route(agt) comp_def = self.replication_comp.replicas[candidate_comp] algo = comp_def.algo.algo algo_module = import_module('pydcop.algorithms.{}'.format(algo)) communication_load = algo_module.communication_load msg_load = 0 for l in comp_def.node.neighbors: if l == neighbor_comp: msg_load += communication_load(comp_def.node, neighbor_comp) com_load = msg_load * route_cost return com_load # Now that we have the variables and constraints, we can create # computation instances for each of the variable this agent is # responsible for, i.e. the binary variables x_i^m that correspond to # the candidate variable x_i (and a_m is the current agent) self._repair_computations.clear() algo_def = AlgoDef(repair_algo.algo_name(), cycle_stop=10, threshold=0.2) for (comp, agt), candidate_var in candidate_binvars.items(): self.logger.debug( 'Building computation for binary variable %s (' 'variable %s on %s)', candidate_var, comp, agt) comm_c = create_agent_comp_comm_constraint(agt, comp, repair_info[comp], comm_func, orphaned_binvars) self.logger.debug( 'Communication constraint for computation %s ' 'on agt %s : %r', comp, self.name, comm_c) constraints = [comm_c, hosting_c, capacity_c, hosted_cs[comp]] # constraints.extend(hosted_cs.values()) self.logger.debug('Got %s Constraints for var %s : %s ', len(constraints), candidate_var, constraints) node = chg.VariableComputationNode(candidate_var, constraints) comp_def = ComputationDef(node, algo_def) computation = repair_algo.build_computation(comp_def) self.logger.debug('Computation for %s : %r ', candidate_var, computation) # add the computation on this agents and register the neighbors self.add_computation(computation) self._repair_computations[computation.name] = \ RepairComputationRegistration(computation, 'ready', comp) for neighbor_comp in node.neighbors: neighbor_agt = agt_hosting_binvar[neighbor_comp] try: self.discovery.register_computation(neighbor_comp, neighbor_agt, publish=False) except UnknownAgent: # If we don't know this agent yet, we must perform a lookup # and only register the computation once found. # Note the use of partial, to force the capture of # neighbor_comp. def _agt_lookup_done(comp, evt, evt_agt, _): if evt == 'agent_added': self.discovery.register_computation(comp, evt_agt, publish=False) self.discovery.subscribe_agent(neighbor_agt, partial( _agt_lookup_done, neighbor_comp), one_shot=True) self.logger.info( 'Repair setup done one %s, %s computations created, ' 'inform orchestrator', self.name, len(candidate_binvars)) return candidate_binvars
def solve(dcop: DCOP, algo_def: Union[str, AlgoDef], distribution: Union[str, Distribution], graph: Union[str, ComputationGraph] = None, timeout=5): """Solve a dcop in a single process. This is simply a convenience method that hides all the complexity of the orchestrator, agents creation, etc. Parameters ---------- dcop : DCOP a DCOP object algo_def: string or AlgoDef object The algorithm that should be used to solve the DCOP. If `algo_def` is a string, it is interpreted as an algorithm module in the package `pydcop.algorithms`, and this algorithm is used with it's default parameters. If `algo_def` is not a string, it must be an AlgoDef object. distribution: either a Distribution object or a string If `distribution` is a string, it is interpreted as the name of a module in the package pydcop.distribution. This module is the loaded and used to generate a distribution of the DCOP on the agents. If `distribution` is not a string, it must be a Distribution object that maps computations to agents. graph: either a string a ComputationGraph object If `graph` is a string, it is interpreted as the name of module in the `pydcop.computations_graph` package ; the module is loaded and used to build computation graph for the dcop. If `graph` is not a string, it must be a ComputationGraph object for the given DCOP. Ths parameter is optional, if it is not given it is deduced from the algorithm. timeout:float in seconds Returns ------- the result Examples -------- assignment = solve(dcop, 'maxsum', 'adhoc', timeout=3) """ if isinstance(algo_def, str): algo_def = AlgoDef(algo_def) algo_module = import_module('pydcop.algorithms.' + algo_def.algo) if graph is None: graph_module = import_module('pydcop.computations_graph.{}'.format( algo_module.GRAPH_TYPE)) graph = graph_module.build_computation_graph(dcop) elif isinstance(graph, str): graph_module = import_module('pydcop.computations_graph.' + graph) graph = graph_module.build_computation_graph(dcop) if isinstance(distribution, str): distrib_module = import_module('pydcop.distribution.' + distribution) distribution = distrib_module.distribute( graph, dcop.agents.values(), computation_memory=algo_module.computation_memory, communication_load=algo_module.communication_load) orchestrator = run_local_thread_dcop(algo_def, graph, distribution, dcop, INFINITY) try: print('Deploy') orchestrator.deploy_computations() print('start Running') orchestrator.run(timeout=timeout) print('Running, wait ready') orchestrator.wait_ready() print('Done') return orchestrator.end_metrics()['assignment'] except Exception: orchestrator.stop_agents(5) orchestrator.stop() finally: orchestrator.stop_agents(5) orchestrator.stop()