Пример #1
0
def distribute(computation_graph: ComputationGraph,
               agentsdef: Iterable[AgentDef],
               hints: DistributionHints = None,
               computation_memory=None,
               communication_load=None) -> Distribution:
    """
    Simplistic distribution method: each computation is hosted on agent 
    agent and each agent host a single computation.
    Agent capacity is not considered.

    Raises an ImpossibleDistributionException
    
    :param computation_graph: 
    :param agentsdef: AgntsDef object containing the list of agent, there must
    be at least as many agents as computations
    :param hints DistributionHints
     
    :return: a distribution a dict {agent_name: [ var_name, ...]} 
    """

    agents = list(agentsdef)

    if len(agents) < len(computation_graph.nodes):
        raise ImpossibleDistributionException(
            'Not enough agents for one agent for each computation : {} < {}'.
            format(len(agents), len(computation_graph.nodes)))

    agent_names = [a.name for a in agents]
    distribution = defaultdict(lambda: list())
    for n, a in zip(computation_graph.nodes, agent_names):
        distribution[a].append(n.name)

    return Distribution(distribution)
Пример #2
0
def distribute(
    computation_graph: ComputationsFactorGraph,
    agentsdef: Iterable[AgentDef],
    hints=None,
    computation_memory: Callable[[ComputationNode], float] = None,
    communication_load: Callable[[ComputationNode, str], float] = None,
    timeout=600,  # Max 10 min
) -> Distribution:
    if computation_memory is None or communication_load is None:
        raise ImpossibleDistributionException(
            "oilp_secp_fgdp distribution requires "
            "computation_memory and link_communication functions"
        )

    mapping = defaultdict(lambda: [])
    agents_capa = {a.name: a.capacity for a in agentsdef}
    variable_computations, factor_computations = [], []
    for comp in computation_graph.nodes:
        if isinstance(comp, VariableComputationNode):
            variable_computations.append(comp.name)
        elif isinstance(comp, FactorComputationNode):
            factor_computations.append(comp.name)
        else:
            raise ImpossibleDistributionException(
                f"Error: {comp} is neither a factor nor a variable computation"
            )
    # actuators variables and cost factor on the corresponding agent:
    for variable in variable_computations[:]:

        for agent in agentsdef:
            if agent.hosting_cost(variable) == 0:
                # Found an actuator variable, host it on the agent
                mapping[agent.name].append(variable)
                variable_computations.remove(variable)
                agents_capa[agent.name] -= computation_memory(
                    computation_graph.computation(variable)
                )
                # search for the cost factor, if any, and host it on the same agent.
                for factor in factor_computations[:]:
                    if f"c_{variable}" == factor:
                        mapping[agent.name].append(factor)
                        factor_computations.remove(factor)
                        agents_capa[agent.name] -= computation_memory(
                            computation_graph.computation(factor)
                        )
                if agents_capa[agent.name] < 0:
                    raise ImpossibleDistributionException(
                        f"Not enough capacity on {agent} to hosts actuator {variable}: {agents_capa[agent.name]}"
                    )
                break
    logger.info(f"Actuator variables - agents: {dict(mapping)}")
    logger.info(f"Remaining capacity: {dict(agents_capa)}")

    return fg_secp_ilp(
        computation_graph,
        agentsdef,
        Distribution(mapping),
        computation_memory,
        communication_load,
    )
Пример #3
0
def distribute(computation_graph: ComputationGraph,
               agentsdef: Iterable[AgentDef],
               hints: DistributionHints=None,
               computation_memory=None,
               communication_load=None) -> Distribution:
    """
    Generate a distribution for the given computation graph.


    :param computation_graph: a ComputationGraph
    :param agentsdef: agents' definitions
    :param hints: a DistributionHints
    :param computation_memory: a function that takes a computation node and its
    Link node as  arguments and return the memory footprint for this node
    :param communication_load: a function that takes a Link as an argument
      and return the communication cost of this edge
    """

    footprint = footprint_fonc(computation_graph, computation_memory)
    capacity = capacity_fonc(agentsdef)
    route = route_fonc(agentsdef)
    msg_load = msg_load_func(computation_graph, communication_load)
    hosting_cost = hosting_cost_func(agentsdef)

    mapping = lp_model(computation_graph, agentsdef, footprint, capacity, route,
                       msg_load, hosting_cost)
    dist = Distribution(mapping)

    return dist
Пример #4
0
    def test_host_on_new_agent(self):

        d = Distribution({'a1': ['v1', 'v2'], 'a2': ['v3']})
        d.host_on_agent('a3', ['v4'])

        self.assertEqual(d.agent_for('v4'), 'a3')

        self.assertIn('a3', d.agents)
        self.assertIn('v4', d.computations_hosted('a3'))
Пример #5
0
    def test_is_hosted_several_computation(self):
        d = Distribution({'a1': ['v1'], 'a2': ['v2']})

        self.assertTrue(d.is_hosted(['v1', 'v2']))
        self.assertTrue(d.is_hosted(['v2', 'v1']))

        self.assertFalse(d.is_hosted(['v3']))
        self.assertFalse(d.is_hosted(['v1', 'v3']))
        self.assertFalse(d.is_hosted(['v3', 'v2']))
Пример #6
0
def load_dist(dist_str: str) -> Distribution:
    loaded = yaml.load(dist_str)

    if 'distribution' not in loaded:
        raise ValueError('Invalid distribution file')

    loaded_dist = loaded['distribution']

    return Distribution(loaded_dist)
Пример #7
0
    def test_dist(self):

        d = Distribution({'a1': ['v1'], 'a2': ['v2']})

        self.assertEqual(len(d.computations_hosted('a1')), 1)
        self.assertEqual(len(d.computations_hosted('a2')), 1)
        self.assertIn('v1', d.computations_hosted('a1'))
        self.assertIn('v2', d.computations_hosted('a2'))

        self.assertEqual(d.agent_for('v1'), 'a1')
        self.assertEqual(d.agent_for('v2'), 'a2')
Пример #8
0
def distribute(
        computation_graph: ComputationGraph,
        agentsdef: Iterable[AgentDef],
        hints: DistributionHints = None,
        computation_memory: Callable[[ComputationNode], float] = None,
        communication_load: Callable[[ComputationNode, str], float] = None,
        timeout=None,  # not used
) -> Distribution:
    if computation_memory is None:
        raise ImpossibleDistributionException("adhoc distribution requires "
                                              "computation_memory functions")

    mapping = defaultdict(lambda: [])
    agents_capa = {a.name: a.capacity for a in agentsdef}
    computations = computation_graph.node_names()
    # as we're dealing with a secp modelled as a constraint graph,
    # we only have actuator and pysical model variables.

    # First, put each actuator variable on its agent
    for agent in agentsdef:
        for comp in computation_graph.node_names():
            if agent.hosting_cost(comp) == 0:
                mapping[agent.name].append(comp)
                computations.remove(comp)
                agents_capa[agent.name] -= computation_memory(
                    computation_graph.computation(comp))
                if agents_capa[agent.name] < 0:
                    raise ImpossibleDistributionException(
                        f"Not enough capacity on {agent} to hosts actuator {comp}: {agents_capa[agent.name]}"
                    )
                break
    logger.info(f"Actuator variables on agents: {dict(mapping)}")

    # We must now place physical model variable on an agent that host
    # a variable it depends on.
    # As physical models always depends on actuator variable,
    # there must always be a computation it depends on that is already hosted.

    for comp in computations:
        footprint = computation_memory(computation_graph.computation(comp))
        neighbors = computation_graph.neighbors(comp)

        candidates = find_candidates(agents_capa, comp, footprint, mapping,
                                     neighbors)

        # Host the computation on the first agent and decrease its remaining capacity
        selected = candidates[0][2]
        mapping[selected].append(comp)
        agents_capa[selected] -= footprint

    return Distribution({a: list(mapping[a]) for a in mapping})
Пример #9
0
def distribute(computation_graph: ComputationGraph,
               agentsdef: Iterable[AgentDef],
               hints: DistributionHints = None,
               computation_memory=None,
               communication_load=None,
               timeout=None) -> Distribution:
    """
    Simplistic distribution method: each computation is hosted on agent 
    agent and each agent host a single computation.
    Agent capacity is not considered.

    Raises an ImpossibleDistributionException

    Parameters
    ----------
    computation_graph: a ComputationGraph
         the computation graph containing the computation that must be
         distributed
    agentsdef: iterable of AgentDef objects
        The definition of the agents the computation will be assigned to.
        There **must** be at least as many agents as computations.
    hints:
        Not used by the ``oneagent`` distribution method.
    computation_memory:
        Not used by the ``oneagent`` distribution method.
    computation_memory:
        Not used by the ``oneagent`` distribution method.

    Returns
    -------
    distribution: Distribution
        A Distribution object containing the mapping form agents to
        computations.
    """

    agents = list(agentsdef)

    if len(agents) < len(computation_graph.nodes):
        raise ImpossibleDistributionException(
            'Not enough agents for one agent for each computation : {} < {}'.
            format(len(agents), len(computation_graph.nodes)))

    agent_names = [a.name for a in agents]
    distribution = defaultdict(lambda: list())
    for n, a in zip(computation_graph.nodes, agent_names):
        distribution[a].append(n.name)

    return Distribution(distribution)
Пример #10
0
def distribute(
        computation_graph: ComputationConstraintsHyperGraph,
        agentsdef: Iterable[AgentDef],
        hints=None,
        computation_memory: Callable[[ComputationNode], float] = None,
        communication_load: Callable[[ComputationNode, str], float] = None,
        timeout=600,  # Max 10 min
) -> Distribution:
    if computation_memory is None or communication_load is None:
        raise ImpossibleDistributionException(
            "oilp_secp_cgdp distribution requires "
            "computation_memory and link_communication functions")

    mapping = defaultdict(lambda: [])
    agents_capa = {a.name: a.capacity for a in agentsdef}
    computations = computation_graph.node_names()
    # as we're dealing with a secp modelled as a constraint graph,
    # we only have actuator and pysical model variables.

    # First, put each actuator variable on its agent
    for agent in agentsdef:
        for comp in computation_graph.node_names():
            if agent.hosting_cost(comp) == 0:
                mapping[agent.name].append(comp)
                computations.remove(comp)
                agents_capa[agent.name] -= computation_memory(
                    computation_graph.computation(comp))
                if agents_capa[agent.name] < 0:
                    raise ImpossibleDistributionException(
                        f"Not enough capacity on {agent} to hosts actuator {comp}: {agents_capa[agent.name]}"
                    )

    logger.info(f"Actuator variables on agents: {dict(mapping)}")
    logger.info(f"Remaining capacity: {dict(agents_capa)}")

    return cg_secp_ilp(
        computation_graph,
        agentsdef,
        Distribution(mapping),
        computation_memory,
        communication_load,
    )
Пример #11
0
def distribute(
    computation_graph: ComputationGraph,
    agentsdef: Iterable[AgentDef],
    hints: DistributionHints = None,
    computation_memory=None,
    communication_load=None,
) -> Distribution:
    """

    Parameters
    ----------
    computation_graph
    agentsdef
    hints
    computation_memory
    communication_load

    Returns
    -------

    """
    footprint_f = footprint_fonc(computation_graph, computation_memory)
    capacity_f = capacity_fonc(agentsdef)
    route_f = route_fonc(agentsdef)
    msg_load_f = msg_load_func(computation_graph, communication_load)
    hosting_cost_f = hosting_cost_func(agentsdef)

    return Distribution(
        ilp_cgdp(
            computation_graph,
            agentsdef,
            footprint_f,
            capacity_f,
            route_f,
            msg_load_f,
            hosting_cost_f,
        ))
Пример #12
0
    def _dump_repair_metrics(self, repair_status, repair_duration ):
        # Dump current distribution
        dist = {a: self.discovery.agent_computations(a)
                for a in self.discovery.agents()}
        result = {
            'inputs': {
                'dist_algo': 'repair',
            },
            "duration": repair_duration,
            'distribution': dist,
            "metrics": self.repair_metrics,
            "status": repair_status
        }

        try:
            cost, comm, hosting = gh_cgdp.distribution_cost(
                Distribution(dist),
                self.graph,
                self._dcop.agents.values(),  # AgentDef s
                computation_memory=self._algo_module.computation_memory,
                communication_load=self._algo_module.communication_load,
            )
            result["cost"] = cost
            result["communication_cost"] = comm
            result["hosting_cost"] = hosting
        except Exception as e:
            self.logger.error("Could not distribute ")
            cost, comm, hosting = None, None, None
            result["cost"] = None
            result["communication_cost"] = None
            result["hosting_cost"] = None
            result["cost_error"] = str(e)

        f_name = 'evtdist_{}.yaml'.format(self.dist_count)
        with open(f_name, mode='w', encoding='utf-8') as f:
            f.write(yaml.dump(result))
Пример #13
0
def _distribute_try(computation_graph: ComputationGraph,
                    agents: Iterable[AgentDef],
                    hints: DistributionHints = None,
                    computation_memory=None,
                    communication_load=None,
                    attempt=0):

    agents_capa = {a.name: a.capacity for a in agents}
    # The distribution methods depends on the order used to process the node,
    # we shuffle them to test a new configuration when retry a distribution
    # after a failure
    nodes = list(computation_graph.nodes)
    shuffle(nodes)
    mapping = defaultdict(set)
    var_hosted = {}

    # Distribute owned computation variable on the corresponding agent.
    # For dcop build from an secp, this is the same thing as deploying the
    # light variable on the light devices, as we were doing before.
    for a in agents_capa:
        for c in hints.must_host(a):
            mapping[a].add(c)
            var_hosted.update({c: a})
            agents_capa[a] -= computation_memory(
                computation_graph.computation(c))

    # First mimic original secp adhoc behavior
    for n in nodes:
        if n.name in var_hosted:
            continue
        hostwith = hints.host_with(n.name)
        # secp models have a constraint that should be hosted on the same
        # agent than the variable of the model
        if len(hostwith) == 1 and n.type == 'FactorComputation' and \
            computation_graph.computation(hostwith[0]).type \
                == 'VariableComputation':

            dependent_var = [v.name for v in n.factor.dimensions]
            candidates = [
                a for a in agents_capa
                if len(set(mapping[a]).intersection(dependent_var)) > 0
            ]

            candidates.sort(key=lambda x: len(mapping[a]))
            if candidates:
                selected = candidates[0]
            else:
                selected = choice(list(agents_capa.keys()))

            mapping[selected].update({n.name, hostwith[0]})
            var_hosted[n.name] = selected
            var_hosted[hostwith[0]] = selected
            agents_capa[selected] -= computation_memory(n)

    for n in nodes:
        if n.name in var_hosted:
            continue
        footprint = computation_memory(n)
        # Candidates : hints only with enough capacity
        candidates = [(agents_capa[a], a) for a in hints.host_with(n.name)
                      if agents_capa[a] > footprint]
        # If no hinted agents has enough capacity, fall back to all agents
        if not candidates:
            candidates = [(c, a) for a, c in agents_capa.items()
                          if c > footprint]

        # Select the candidate that is already hosting the highest
        # number of computations sharing a link with this one.
        scores = []
        for capacity, a in candidates:
            count = 0
            for l in computation_graph.links_for_node(n.name):
                count += len([None for l_n in l.nodes if l_n in mapping[a]])
            # The tuple is in this order so that we sort by score first,
            # and then by available capacity.
            scores.append((count, capacity, a))
        scores.sort(reverse=True)

        if scores:
            selected = scores[0][2]
            agents_capa[selected] -= footprint
        else:
            # Retry 3 times in case of failure, the nodes will be shuffled
            # every time, increasing the probability to find a feasible
            # distribution.
            if attempt > 2:
                raise ImpossibleDistributionException(
                    'Could not find feasible distribution after {} '
                    'attempts'.format(attempt))
            else:
                _distribute_try(computation_graph, agents, hints,
                                computation_memory, computation_graph,
                                attempt + 1)

        mapping[selected].update({n.name})
        var_hosted[n.name] = selected

    return Distribution({a: list(mapping[a]) for a in mapping})
Пример #14
0
def distribute(
        computation_graph: ComputationsFactorGraph,
        agentsdef: Iterable[AgentDef],
        hints: DistributionHints = None,
        computation_memory: Callable[[ComputationNode], float] = None,
        communication_load: Callable[[ComputationNode, str], float] = None,
        timeout=None,  # not used
) -> Distribution:
    if computation_memory is None:
        raise ImpossibleDistributionException("adhoc distribution requires "
                                              "computation_memory functions")

    # as we're dealing with a secp modelled as a factor graph, we have computations for
    # actuator and physical model variables, rules and physical model factors.
    mapping = defaultdict(lambda: [])
    agents_capa = {a.name: a.capacity for a in agentsdef}
    variable_computations = []
    factor_computations = []
    for comp in computation_graph.nodes:
        if isinstance(comp, VariableComputationNode):
            variable_computations.append(comp.name)
        elif isinstance(comp, FactorComputationNode):
            factor_computations.append(comp.name)
        else:
            raise ImpossibleDistributionException(
                f"Error: {comp} is neither a factor nor a variable computation"
            )

    # First, put each actuator variable and cost factor on its agent
    for variable in variable_computations[:]:

        for agent in agentsdef:
            if agent.hosting_cost(variable) == 0:
                # Found an actuator variable, host it on the agent
                mapping[agent.name].append(variable)
                variable_computations.remove(variable)
                agents_capa[agent.name] -= computation_memory(
                    computation_graph.computation(variable))
                # search for the cost factor, if any, and host it on the same agent.
                for factor in factor_computations[:]:
                    if f"c_{variable}" == factor:
                        mapping[agent.name].append(factor)
                        factor_computations.remove(factor)
                        agents_capa[agent.name] -= computation_memory(
                            computation_graph.computation(factor))
                if agents_capa[agent.name] < 0:
                    raise ImpossibleDistributionException(
                        f"Not enough capacity on {agent} to hosts actuator {variable}: {agents_capa[agent.name]}"
                    )
                break
    logger.info(f"Actuator computations - agents: {dict(mapping)}")
    logger.info(f"Remaining capacity: {dict(agents_capa)}")

    # now find computations for physical models and variables variables.
    # * all remaining variables are model variables
    # * physical model factor computation names contain the name of the variable
    model_variables = variable_computations
    models = []
    for model_var in model_variables:
        for fact in factor_computations:
            if f"c_{model_var}" == fact:
                models.append((model_var, fact))
                factor_computations.remove(fact)

    # All remaining factor ar rule factors
    rule_factors = factor_computations

    logger.debug(f"Physical models: {models}")
    logger.debug(f"Rules: {rule_factors}")

    # Now place models
    for model_var, model_fac in models:
        footprint = computation_memory(
            computation_graph.computation(model_fac)) + computation_memory(
                computation_graph.computation(model_var))
        neighbors = computation_graph.neighbors(model_fac)

        candidates = find_candidates(agents_capa, model_fac, footprint,
                                     mapping, neighbors)

        # Host the model on the first agent and decrease its remaining capacity
        selected = candidates[0][2]
        mapping[selected].append(model_var)
        mapping[selected].append(model_fac)
        agents_capa[selected] -= footprint
    logger.debug(f"All models hosted: {dict(mapping)}")
    logger.debug(f"Remaining capacity: {agents_capa}")

    # And rules at last:
    for rule_fac in rule_factors:
        footprint = computation_memory(computation_graph.computation(rule_fac))
        neighbors = computation_graph.neighbors(rule_fac)

        candidates = find_candidates(agents_capa, rule_fac, footprint, mapping,
                                     neighbors)

        # Host the computation on the first agent and decrease its remaining capacity
        selected = candidates[0][2]
        mapping[selected].append(rule_fac)
        agents_capa[selected] -= footprint

    return Distribution({a: list(mapping[a]) for a in mapping})
Пример #15
0
def distribute(
    computation_graph: ComputationGraph,
    agentsdef: Iterable[AgentDef],
    hints=None,
    computation_memory: Callable[[ComputationNode], float] = None,
    communication_load: Callable[[ComputationNode, str], float] = None,
) -> Distribution:
    """
    gh-cgdp distribution method.

    Heuristic distribution baed on communication and hosting costs, while respecting
    agent's capacities

    Parameters
    ----------
    computation_graph
    agentsdef
    hints
    computation_memory
    communication_load

    Returns
    -------
    Distribution:
        The distribution for the computation graph.

    """

    # Place computations with hosting costs == 0
    # For SECP, this assign actuators var and factor to the right device.
    fixed_mapping = {}
    for comp in computation_graph.node_names():
        for agent in agentsdef:
            if agent.hosting_cost(comp) == 0:
                fixed_mapping[comp] = (
                    agent.name,
                    computation_memory(computation_graph.computation(comp)),
                )
                break

    # Sort computation by footprint, but add a random element to avoid sorting on names
    computations = [(computation_memory(n), n, None, random.random())
                    for n in computation_graph.nodes
                    if n.name not in fixed_mapping]
    computations = sorted(computations,
                          key=lambda o: (o[0], o[3]),
                          reverse=True)
    computations = [t[:-1] for t in computations]
    logger.info("placing computations %s",
                [(f, c.name) for f, c, _ in computations])

    current_mapping = {}  # Type: Dict[str, str]
    i = 0
    while len(current_mapping) != len(computations):
        footprint, computation, candidates = computations[i]
        logger.debug(
            "Trying to place computation %s with footprint %s",
            computation.name,
            footprint,
        )
        # look for cancidiate agents for computation c
        # TODO: keep a list of remaining capacities for agents ?
        if candidates is None:
            candidates = candidate_hosts(
                computation,
                footprint,
                computations,
                agentsdef,
                communication_load,
                current_mapping,
                fixed_mapping,
            )
            computations[i] = footprint, computation, candidates
        logger.debug("Candidates for computation %s : %s", computation.name,
                     candidates)

        if not candidates:
            if i == 0:
                logger.error(
                    f"Cannot find a distribution, no candidate for computation {computation}\n"
                    f" current mapping: {current_mapping}")
                raise ImpossibleDistributionException(
                    f"Impossible Distribution, no candidate for {computation}")

            # no candidate : backtrack !
            i -= 1
            logger.info(
                "No candidate for %s, backtrack placement "
                "of computation %s (was on %s",
                computation.name,
                computations[i][1].name,
                current_mapping[computations[i][1].name],
            )
            current_mapping.pop(computations[i][1].name)

            # FIXME : eliminate selected agent for previous computation
        else:
            _, selected = candidates.pop()
            current_mapping[computation.name] = selected.name
            computations[i] = footprint, computation, candidates
            logger.debug("Place computation %s on agent %s", computation.name,
                         selected.name)
            i += 1

    # Build the distribution for the mapping
    agt_mapping = defaultdict(lambda: [])
    for c, a in current_mapping.items():
        agt_mapping[a].append(c)
    for c, (a, _) in fixed_mapping.items():
        agt_mapping[a].append(c)
    dist = Distribution(agt_mapping)

    return dist
Пример #16
0
def factor_graph_lp_model(cg: ComputationsFactorGraph,
                          agents: List[AgentDef],
                          must_host: Dict[str, List],
                          computation_memory=None,
                          communication_load=None):
    """
    To distribute we need:
    * com : the communication cost of an edge between a var and a fact
    * mem_var : the memory footprint of a variable computation
    * mem_fac : the memory footprint of a factor computation
    
    These function depends on the algorithm.

    Here    
    * mem_var and mem_fac are given by the computation_memory method.
    * com is given by computation_memory

    :return:
    """
    variables = [n for n in cg.nodes if n.type == 'VariableComputation']
    factors = [n for n in cg.nodes if n.type == 'FactorComputation']

    agents = list(agents)
    agents_names = [a.name for a in agents]

    fixed_dist = Distribution(must_host)

    # Only keep computations for which we actually need to find an agent.
    vars_to_host = [
        v.name for v in variables if not fixed_dist.has_computation(v.name)
    ]
    facs_to_host = [
        f.name for f in factors if not fixed_dist.has_computation(f.name)
    ]

    # x_i^k : binary variable indicating if var x_i is hosted on agent a_k.
    xs = _build_xs_binvar(vars_to_host, agents_names)
    # f_j^k : binary variable indicating if factor f_j is hosted on agent a_k.
    fs = _build_fs_binvar(facs_to_host, agents_names)
    # alpha_ijk : binary variable indicating if  x_i and f_j are both on a_k.
    alphas = _build_alphaijk_binvars(cg, agents_names)

    # LP problem with objective function (total communication cost).
    pb = LpProblem('distribution', LpMinimize)
    pb += _objective_function(cg, communication_load, alphas,
                              agents_names), 'Communication costs'
    # Constraints.
    # All variable computations must be hosted:
    for i in vars_to_host:
        pb += lpSum([xs[(i, k)] for k in agents_names]) == 1, \
              'var {} is hosted'.format(i)

    # All factor computations must be hosted:
    for j in facs_to_host:
        pb += lpSum([fs[(j, k)] for k in agents_names]) == 1, \
              'factor {} is hosted'.format(j)

    # Each agent must host at least one computation:
    # We only need this constraints for agents that do not already host a
    # computation:
    empty_agents = [a for a in agents_names if not must_host[a]]
    for k in empty_agents:
        pb += lpSum([xs[(i, k)] for i in vars_to_host]) + \
              lpSum([fs[(j, k)] for j in facs_to_host]) >= 1, \
              'atleastone {}'.format(k)

    # Memory capacity constraint for agents
    for a in agents:
        # Decrease capacity for already hosted computations
        capacity = a.capacity - \
                   sum([_computation_memory_in_cg(c, cg, computation_memory)
                        for c in must_host[a.name]])

        pb += lpSum([_computation_memory_in_cg(i, cg, computation_memory) *
                     xs[(i, a.name)] for i in vars_to_host]) \
            + lpSum([_computation_memory_in_cg(j, cg, computation_memory) *
                     fs[(j, a.name)] for j in facs_to_host]) <= capacity, \
            'memory {}'.format(a.name)

    # Linearization constraints for alpha_ijk.
    for link in cg.links:
        i, j = link.variable_node, link.factor_node
        for k in agents_names:

            if i in vars_to_host and j in facs_to_host:
                pb += alphas[((i, j), k)] <= xs[(i, k)], \
                    'lin1 {}{}{}'.format(i, j, k)
                pb += alphas[((i, j), k)] <= fs[(j, k)], \
                    'lin2 {}{}{}'.format(i, j, k)
                pb += alphas[((i, j), k)] >= xs[(i, k)] + fs[(j, k)] - 1, \
                    'lin3 {}{}{}'.format(i, j, k)

            elif i in vars_to_host and j not in facs_to_host:
                # Var is free, factor is already hosted
                if fixed_dist.agent_for(j) == k:
                    pb += alphas[((i, j), k)] == xs[(i, k)]
                else:
                    pb += alphas[((i, j), k)] == 0

            elif i not in vars_to_host and j in facs_to_host:
                # if i is not in vars_vars_to_host, it means that it's a
                # computation that is already hosted (from  hints)
                if fixed_dist.agent_for(i) == k:
                    pb += alphas[((i, j), k)] == fs[(j, k)]
                else:
                    pb += alphas[((i, j), k)] == 0

            else:
                # i and j are both alredy hosted
                if fixed_dist.agent_for(i) == k and fixed_dist.agent_for(j) \
                        == k:
                    pb += alphas[((i, j), k)] == 1
                else:
                    pb += alphas[((i, j), k)] == 0

    # Now solve our LP
    # status = pb.solve(GLPK_CMD())
    # status = pb.solve(GLPK_CMD(mip=1))
    # status = pb.solve(GLPK_CMD(mip=0, keepFiles=1,
    #                                options=['--simplex', '--interior']))
    status = pb.solve(GLPK_CMD(keepFiles=0, msg=False, options=['--pcost']))

    if status != LpStatusOptimal:
        raise ImpossibleDistributionException("No possible optimal"
                                              " distribution ")
    else:
        logger.debug('GLPK cost : %s', value(pb.objective))

        comp_dist = fixed_dist
        for k in agents_names:

            agt_vars = [
                i for i, ka in xs if ka == k and value(xs[(i, ka)]) == 1
            ]
            comp_dist.host_on_agent(k, agt_vars)

            agt_rels = [
                j for j, ka in fs if ka == k and value(fs[(j, ka)]) == 1
            ]
            comp_dist.host_on_agent(k, agt_rels)
        return comp_dist
Пример #17
0
 def test_is_hosted_single_computation(self):
     d = Distribution({'a1': ['v1'], 'a2': ['v2']})
     self.assertTrue(d.is_hosted('v1'))
     self.assertTrue(d.is_hosted('v2'))
     self.assertFalse(d.is_hosted('v3'))
Пример #18
0
def generate(args):
    slots, events, resources = generate_problem_definition(
        args.slots_count,
        args.resources_count,
        args.max_resource_value,
        args.events_count,
        args.max_length_event,
        args.max_resources_event,
    )

    penalty = args.max_resource_value * args.slots_count * args.resources_count
    variables, constraints, agents = peav_model(slots, events, resources,
                                                penalty)

    domains = {
        variable.domain.name: variable.domain
        for variable in variables.values()
    }
    variables = {variable.name: variable for variable in variables.values()}
    # agents_defs = {agent.name: agent for agent, _ in agents.values()}
    # Generate agents hosting and route costs
    agents_defs = {}
    if not args.no_agents:
        for agent, agt_variables in agents.items():
            kw = {}
            kw["hosting_costs"] = {v.name: 0 for v in agt_variables}
            if args.hosting_default:
                kw["default_hosting_cost"] = args.hosting_default
            if args.capacity:
                kw["capacity"] = args.capacity
            if args.routes_default:
                kw["default_route"] = args.routes_default
            agents_defs[agent] = AgentDef(agent, **kw)

    dcop = DCOP(
        "MeetingSceduling",
        objective="max",
        domains=domains,
        variables=variables,
        constraints=constraints,
        agents=agents_defs,
    )

    if not args.no_agents:
        distribution = Distribution({
            agent.name: [v.name for v in agents[agent.name]]
            for agent in agents_defs.values()
        })

    if args.output:
        output_file = args.output
        with open(output_file, encoding="utf-8", mode="w") as fo:
            fo.write(dcop_yaml(dcop))

        if not args.no_agents:
            dist_result = {
                "inputs": {
                    "dist_algo": "peav",
                    "dcop": output_file,
                    "graph": "constraints_graph",
                    "algo": "NA",
                },
                "distribution": distribution.mapping(),
                "cost": None,
            }
            path, ext = splitext(output_file)
            dist_output_file = f"{path}_dist{ext}"
            with open(dist_output_file, encoding="utf-8", mode="w") as fo:
                fo.write(yaml.dump(dist_result))

    else:
        print(dcop_yaml(dcop))

        if not args.no_agents:
            dist_result = {
                "inputs": {
                    "dist_algo": "peav",
                    "dcop": "NA",
                    "graph": "constraints_graph",
                    "algo": "NA",
                },
                "distribution": distribution.mapping(),
                "cost": None,
            }
            # FIXME proper serialization of the distribution:
            print(yaml.dump(dist_result))
Пример #19
0
 def test_host_on_agent_raises_on_already_hosted_comp(self):
     d = Distribution({'a1': ['v1', 'v2'], 'a2': ['v3']})
     self.assertRaises(ValueError, d.host_on_agent, 'a1', ['v3'])
Пример #20
0
def distribute(computation_graph: ComputationGraph,
               agentsdef: Iterable[AgentDef],
               hints: DistributionHints=None,
               computation_memory: Callable[[ComputationNode], float]=None,
               communication_load: Callable[[ComputationNode, str],
                                            float]=None) \
        -> Distribution:
    """

    Parameters
    ----------
    computation_graph
    agentsdef
    hints
    computation_memory
    communication_load

    Returns
    -------

    """
    computations = sorted([(computation_memory(n), n, None)
                           for n in computation_graph.nodes],
                          key=lambda o: (o[0], o[1].name),
                          reverse=True)
    logger.info('placing computations %s',
                [(f, c.name) for f, c, _ in computations])

    current_mapping = {}  # Type: Dict[str, str]
    i = 0
    while len(current_mapping) != len(computations):
        footprint, computation, candidates = computations[i]
        logger.debug('Trying to place computation %s with footprint %s',
                     computation.name, footprint)
        # try
        # look for agent for computation c
        if candidates is None:
            candidates = candidate_hosts(computation, footprint, computations,
                                         agentsdef, communication_load,
                                         current_mapping)
            computations[i] = footprint, computation, candidates
        logger.debug('Candidates for computation %s : %s', computation.name,
                     candidates)

        if not candidates:
            if i == 0:
                raise ValueError('Impossible Distribution !')

            # no candidate : backtrack !
            i -= 1
            logger.info(
                'No candidate for %s, backtrack placement '
                'of computation %s (was on %s', computation.name,
                computations[i][1].name,
                current_mapping[computations[i][1].name])
            current_mapping.pop(computations[i][1].name)

            # FIXME : eliminate selected agent for previous computation
        else:
            _, selected = candidates.pop()
            current_mapping[computation.name] = selected.name
            computations[i] = footprint, computation, candidates
            logger.debug('Place computation %s on agent %s', computation.name,
                         selected.name)
            i += 1

    # Build the distribution for the mapping
    agt_mapping = defaultdict(lambda: [])
    for c, a in current_mapping.items():
        agt_mapping[a].append(c)
    dist = Distribution(agt_mapping)

    return dist
Пример #21
0
def generate_iot(args):
    print("generate iot ", args.output)

    # Constraints and variables with a power-law constraint graph:
    variables, constraints, domain = generate_powerlaw_var_constraints(
        args.num, args.domain, args.range
    )

    # Build a dcop and computation graph with no agents, just to be able to
    # compute the footprint of computations:
    dcop = DCOP(
        "graph coloring",
        "min",
        domains={"d": domain},
        variables=variables,
        agents={},
        constraints=constraints,
    )
    graph_module = import_module("pydcop.computations_graph.factor_graph")
    cg = graph_module.build_computation_graph(dcop)
    algo_module = load_algorithm_module("maxsum")

    footprints = {c.name: algo_module.computation_memory(c) for c in cg.nodes}

    # Generate an agent for each variable computation and assign the
    # computation to that agent.
    agents = {}  # type: Dict[str, AgentDef]
    mapping = defaultdict(lambda: [])  # type: Dict[str, List[str]]
    for comp in cg.nodes:
        if isinstance(comp, VariableComputationNode):
            a_name = agt_name(comp.name)
            agt = AgentDef(
                a_name,
                capacity=footprints[comp.name] * 100,
                default_hosting_cost=10,
                hosting_costs=agt_hosting_costs(comp, cg),
                default_route=1,
                routes=agt_route_costs(comp, cg),
            )
            logger.debug(
                "Create agent %s for computation %s with capacity %s",
                agt.name,
                comp.name,
                agt.capacity,
            )
            agents[agt.name] = agt
            mapping[agt.name].append(comp.name)

    # Now, we have created all the agents and distributed all the variables
    # let's distribute the factor computations.
    msg_load = msg_load_func(cg, algo_module.communication_load)
    factor_mapping = distribute_factors(agents, cg, footprints, mapping, msg_load)
    for a in mapping:
        mapping[a].extend(factor_mapping[a])

    dcop = DCOP(
        "graph coloring",
        "min",
        domains={"d": domain},
        variables=variables,
        agents=agents,
        constraints=constraints,
    )

    distribution = Distribution(mapping)

    if args.output:
        outputfile = args.output
        write_in_file(outputfile, dcop_yaml(dcop))

        dist = distribution.mapping()
        cost = ilp_compref.distribution_cost(
            distribution,
            cg,
            dcop.agents.values(),
            computation_memory=algo_module.computation_memory,
            communication_load=algo_module.communication_load,
        )

        result = {
            "inputs": {
                "dist_algo": "io_problem",
                "dcop": args.output,
                "graph": "factor_graph",
                "algo": "maxsum",
            },
            "distribution": dist,
            "cost": cost,
        }
        outputfile = "dist_" + args.output
        write_in_file(outputfile, yaml.dump(result))
    else:
        print(dcop_yaml(dcop))