def on_replicate_answer(self, budget: float, spent: float, rq_path: Path,
paths: PathsTable, visited: List[AgentName],
comp_def: ComputationDef, footprint: float,
replica_count: int, hosts: List[str]):
*_, current, sender = rq_path
initial_path = rq_path[:-1]
paths = filter_missing_agents_paths(
paths,
self.replication_neighbors() | {self.agt_name})
# If all replica have been placed, report back to requester if any, or
# signal that replication is done.
if replica_count == 0:
if len(rq_path) >= 3:
self.logger.debug(
'All replica placed for %s, report back to requester',
comp_def.name)
self._send_answer(budget, spent, initial_path, paths, visited,
comp_def, footprint, replica_count, hosts)
return
else:
self.computation_replicated(comp_def.name, hosts)
return
# If there are still replica to be placed, keep trying on neighbors
back_path = rq_path[:-1]
affordable_paths = affordable_path_from(back_path, budget + spent,
paths)
# Paths to next candidates, avoiding the path we're coming from.
target_paths = (back_path + Path(p.head()) for _, p in affordable_paths
if back_path + Path(p.head()) != rq_path)
for target_path in target_paths:
forwarded, replica_count = \
self._visit_path(budget, spent, target_path, paths, visited,
comp_def, footprint, replica_count, hosts)
if forwarded:
return
# Could not send to any neighbor: get back to requester
if len(rq_path) >= 3:
self._send_answer(budget, spent, initial_path, paths, visited,
comp_def, footprint, replica_count, hosts)
return
# no reachable candidate path and no ancestor to go back,
# we are back at the start node: increase the budget
if not paths:
# Cannot increase budget, replica distribution is finished for
# this computation, even if we have not reached target resiliency
# level. Report the final replica distribution to the orchestrator.
self.computation_replicated(comp_def.name, hosts)
else:
budget = min(c for p, c in paths.items() if p != rq_path)
self.logger.info('Increase budget for computation %s : %s',
comp_def.name, budget)
self.on_replicate_request(budget, 0, Path(current), paths, visited,
comp_def, footprint, replica_count,
hosts)
def on_replicate_request(self, budget: float, spent: float, rq_path: Path,
paths: PathsTable, visited: List[AgentName],
comp_def: ComputationDef, footprint: float,
replica_count: int, hosts: List[str]):
assert self.agt_name == rq_path.last()
if rq_path in paths:
paths.pop(rq_path)
if self.agt_name not in visited: # first visit for this node
visited.append(self.agt_name)
self._add_hosting_path(spent, comp_def.name, rq_path, paths)
neighbors = self.replication_neighbors()
# If some agents have left during replication, some may still be in
# the received paths table even though sending replication_request to
# them would block the replication.
paths = filter_missing_agents_paths(paths, neighbors | {self.agt_name})
# Available & affordable with current remaining budget, paths from here:
affordable_paths = affordable_path_from(rq_path, budget + spent, paths)
# self.logger.debug('Affordable path %s %s %s %s \n%s', budget, spent,
# rq_path, affordable_paths, paths)
# Paths to next candidates from paths table.
target_paths = (rq_path + Path(p.head()) for _, p in affordable_paths)
for target_path in target_paths:
forwarded, replica_count = \
self._visit_path(budget, spent, target_path, paths, visited,
comp_def, footprint, replica_count, hosts)
if forwarded:
return
self.logger.info('No reachable path for %s with budget %s ',
comp_def.name, budget)
# Either:
# * No path : Not on a already known path
# * or all paths are too expensive for our current budget (meaning
# that we are at the last node in the known path)
# In both cases, we now look at neighbors costs and store them if we
# do not already known a cheaper path to them.
neighbors_path = ((n, self.route(n), rq_path + Path(n))
for n in neighbors if n not in visited)
for n, r, p in neighbors_path:
cheapest, cheapest_path = cheapest_path_to(n, paths)
if cheapest > spent + r:
if cheapest_path in paths:
paths.pop(cheapest_path)
paths[p] = spent + r
else:
# self.logger.debug('Cheaper path known to %s : %s (%s)', p,
# cheapest_path, cheapest)
pass
self._send_answer(budget, spent, rq_path, paths, visited, comp_def,
footprint, replica_count, hosts)
def test_filter_missing_agents_paths():
paths = [
(4, ("a2", "a3", "__hosting__")),
(3, ("a2", "a5", "a6")),
(1, ("a3", "a4")),
(3, ("a5", "a3")),
(3, ("a1", "a4", "a2")),
]
removed = {"a4", "a6"}
filtered = filter_missing_agents_paths(paths, removed)
assert len(filtered) == 2
def test_filter_missing_agents_paths():
paths = {
Path('_replication_a2', '_replication_a3', '__hosting__'): 4,
Path('_replication_a2', '_replication_a5', '_replication_a6'): 3,
Path('_replication_a3', '_replication_a4'): 1,
Path('_replication_a5', '_replication_a3'): 3,
Path('_replication_a1', '_replication_a4', '_replication_a2'): 3,
}
available = {
'_replication_a2', '_replication_a3', '_replication_a5',
'_replication_a6'
}
filtered = filter_missing_agents_paths(paths, available)
assert len(filtered) == 3
def test_filter_missing_agents_paths_2():
paths = [
(1, ("a28", "a16")),
(1, ("a28", "a22")),
(1, ("a28", "a32")),
(1, ("a28", "a35")),
(1, ("a28", "a02")),
(1, ("a28", "a20")),
(1, ("a28", "a06")),
(1, ("a28", "a03")),
(1, ("a28", "a26")),
(1, ("a28", "a14")),
(1, ("a28", "a24")),
(11, ("a28", "a18", "__hosting__")),
]
removed = {"a48"}
filtered = filter_missing_agents_paths(paths, removed)
assert len(filtered) == len(paths)
def to_bench():
paths = [
(1, ("a28", "a16")),
(1, ("a28", "a22")),
(1, ("a28", "a32")),
(1, ("a28", "a35")),
(1, ("a28", "a02")),
(1, ("a28", "a20")),
(1, ("a28", "a06")),
(1, ("a28", "a03")),
(1, ("a28", "a26")),
(1, ("a28", "a14")),
(1, ("a28", "a24")),
(11, ("a28", "a18", "__hosting__")),
]
removed = {"a26", "a14"}
filtered = filter_missing_agents_paths(paths, removed)
assert len(list(filtered)) == len(paths) - 2
def on_replicate_answer(
self,
budget: float,
spent: float,
rq_path: Path,
paths: PathsTable,
visited: List[AgentName],
comp_def: ComputationDef,
footprint: float,
replica_count: int,
hosts: List[str],
):
*_, current, sender = rq_path
comp_name = comp_def.name
initial_path = rq_path[:-1]
if self._removed_agents:
paths = filter_missing_agents_paths(paths, self._removed_agents)
# If all replica have been placed, report back to requester if any, or
# signal that replication is done.
if replica_count == 0:
if len(rq_path) >= 3:
if self.logger.isEnabledFor(logging.DEBUG):
self.logger.debug(
f"All replica placed for {comp_name}, report back to requester"
)
self._send_answer(
budget,
spent,
initial_path,
paths,
visited,
comp_def,
footprint,
replica_count,
hosts,
)
return
else:
self.computation_replicated(comp_name, hosts)
return
# If there are still replica to be placed, keep trying on neighbors
back_path = rq_path[:-1]
# affordable_paths = affordable_path_from(back_path, budget + spent, paths)
# Paths to next candidates, avoiding the path we're coming from.
target_paths = (back_path + (p[0], )
for p in affordable_path_from(back_path, budget +
spent, paths)
if back_path + (p[0], ) != rq_path)
for target_path in target_paths:
if self.logger.isEnabledFor(logging.DEBUG):
self.logger.debug(
f"Visiting for {comp_name} path {target_path}")
forwarded, replica_count = self._visit_path(
budget,
spent,
target_path,
paths,
visited,
comp_def,
footprint,
replica_count,
hosts,
)
if forwarded:
return
# Could not send to any neighbor: get back to requester
if len(rq_path) >= 3:
self._send_answer(
budget,
spent,
initial_path,
paths,
visited,
comp_def,
footprint,
replica_count,
hosts,
)
return
# no reachable candidate path and no ancestor to go back,
# we are back at the start node: increase the budget
if not paths:
# Cannot increase budget, replica distribution is finished for
# this computation, even if we have not reached target resiliency
# level. Report the final replica distribution to the orchestrator.
self.logger.warning(
f"Could not reach target resiliency level for {comp_name}, "
f"replicated on {hosts}")
self.computation_replicated(comp_name, hosts)
else:
budget = min(c for c, p in paths if p != rq_path)
if self.logger.isEnabledFor(logging.INFO):
self.logger.info(
f"Increase budget for computation {comp_name} : {budget}")
self.on_replicate_request(
budget,
0,
(current, ),
paths,
visited,
comp_def,
footprint,
replica_count,
hosts,
)
def on_replicate_request(
self,
budget: float,
spent: float,
rq_path: Path,
paths: PathsTable,
visited: List[AgentName],
comp_def: ComputationDef,
footprint: float,
replica_count: int,
hosts: List[str],
):
assert self.agt_name == rq_path[-1] # last()
comp_name = comp_def.name
remove_path(paths, rq_path)
if self.agt_name not in visited: # first visit for this node
visited.append(self.agt_name)
self._add_hosting_path(spent, comp_name, rq_path, paths)
neighbors = self.replication_neighbors()
# If some agents have left during replication, some may still be in
# the received paths table even though sending replication_request to
# them would block the replication.
if self._removed_agents:
paths = filter_missing_agents_paths(paths, self._removed_agents)
# Available & affordable with current remaining budget, paths from here:
# affordable_paths = affordable_path_from(rq_path, budget + spent, paths)
# self.logger.debug(
# f"Affordable paths for {comp_name} on rq (b:{budget}, s:{spent}, {rq_path}) : "
# f": {affordable_paths} out of {paths}"
# )
# Paths to next candidates from paths table.
target_paths = (rq_path + (p[0], )
for p in affordable_path_from(rq_path, budget +
spent, paths))
for target_path in target_paths:
if self.logger.isEnabledFor(logging.DEBUG):
self.logger.debug(
f"Visiting for {comp_name} path {target_path}")
forwarded, replica_count = self._visit_path(
budget,
spent,
target_path,
paths,
visited,
comp_def,
footprint,
replica_count,
hosts,
)
if forwarded:
return
if self.logger.isEnabledFor(logging.INFO):
self.logger.info("No reachable path for %s with budget %s ",
comp_name, budget)
# Either:
# * No path : Not on a already known path
# * or all paths are too expensive for our current budget (meaning
# that we are at the last node in the known path)
# In both cases, we now look at neighbors costs and store them if we
# do not already known a cheaper path to them.
neighbors_path = ((n, c, rq_path + (n, )) for n, c in neighbors
if n not in visited)
for n, r, p in neighbors_path:
cheapest, cheapest_path = cheapest_path_to(n, paths)
if cheapest > spent + r:
remove_path(paths, cheapest_path)
paths.append((spent + r, p))
paths.sort()
else:
# self.logger.debug('Cheaper path known to %s : %s (%s)', p,
# cheapest_path, cheapest)
pass
self._send_answer(
budget,
spent,
rq_path,
paths,
visited,
comp_def,
footprint,
replica_count,
hosts,
)
