def on_replicate_answer(self, budget: float, spent: float, rq_path: Path,
paths: PathsTable, visited: List[AgentName],
comp_def: ComputationDef, footprint: float,
replica_count: int, hosts: List[str]):
*_, current, sender = rq_path
initial_path = rq_path[:-1]
paths = filter_missing_agents_paths(
paths,
self.replication_neighbors() | {self.agt_name})
# If all replica have been placed, report back to requester if any, or
# signal that replication is done.
if replica_count == 0:
if len(rq_path) >= 3:
self.logger.debug(
'All replica placed for %s, report back to requester',
comp_def.name)
self._send_answer(budget, spent, initial_path, paths, visited,
comp_def, footprint, replica_count, hosts)
return
else:
self.computation_replicated(comp_def.name, hosts)
return
# If there are still replica to be placed, keep trying on neighbors
back_path = rq_path[:-1]
affordable_paths = affordable_path_from(back_path, budget + spent,
paths)
# Paths to next candidates, avoiding the path we're coming from.
target_paths = (back_path + Path(p.head()) for _, p in affordable_paths
if back_path + Path(p.head()) != rq_path)
for target_path in target_paths:
forwarded, replica_count = \
self._visit_path(budget, spent, target_path, paths, visited,
comp_def, footprint, replica_count, hosts)
if forwarded:
return
# Could not send to any neighbor: get back to requester
if len(rq_path) >= 3:
self._send_answer(budget, spent, initial_path, paths, visited,
comp_def, footprint, replica_count, hosts)
return
# no reachable candidate path and no ancestor to go back,
# we are back at the start node: increase the budget
if not paths:
# Cannot increase budget, replica distribution is finished for
# this computation, even if we have not reached target resiliency
# level. Report the final replica distribution to the orchestrator.
self.computation_replicated(comp_def.name, hosts)
else:
budget = min(c for p, c in paths.items() if p != rq_path)
self.logger.info('Increase budget for computation %s : %s',
comp_def.name, budget)
self.on_replicate_request(budget, 0, Path(current), paths, visited,
comp_def, footprint, replica_count,
hosts)
def on_replicate_request(self, budget: float, spent: float, rq_path: Path,
paths: PathsTable, visited: List[AgentName],
comp_def: ComputationDef, footprint: float,
replica_count: int, hosts: List[str]):
assert self.agt_name == rq_path.last()
if rq_path in paths:
paths.pop(rq_path)
if self.agt_name not in visited: # first visit for this node
visited.append(self.agt_name)
self._add_hosting_path(spent, comp_def.name, rq_path, paths)
neighbors = self.replication_neighbors()
# If some agents have left during replication, some may still be in
# the received paths table even though sending replication_request to
# them would block the replication.
paths = filter_missing_agents_paths(paths, neighbors | {self.agt_name})
# Available & affordable with current remaining budget, paths from here:
affordable_paths = affordable_path_from(rq_path, budget + spent, paths)
# self.logger.debug('Affordable path %s %s %s %s \n%s', budget, spent,
# rq_path, affordable_paths, paths)
# Paths to next candidates from paths table.
target_paths = (rq_path + Path(p.head()) for _, p in affordable_paths)
for target_path in target_paths:
forwarded, replica_count = \
self._visit_path(budget, spent, target_path, paths, visited,
comp_def, footprint, replica_count, hosts)
if forwarded:
return
self.logger.info('No reachable path for %s with budget %s ',
comp_def.name, budget)
# Either:
# * No path : Not on a already known path
# * or all paths are too expensive for our current budget (meaning
# that we are at the last node in the known path)
# In both cases, we now look at neighbors costs and store them if we
# do not already known a cheaper path to them.
neighbors_path = ((n, self.route(n), rq_path + Path(n))
for n in neighbors if n not in visited)
for n, r, p in neighbors_path:
cheapest, cheapest_path = cheapest_path_to(n, paths)
if cheapest > spent + r:
if cheapest_path in paths:
paths.pop(cheapest_path)
paths[p] = spent + r
else:
# self.logger.debug('Cheaper path known to %s : %s (%s)', p,
# cheapest_path, cheapest)
pass
self._send_answer(budget, spent, rq_path, paths, visited, comp_def,
footprint, replica_count, hosts)
def _visit_path(self, budget: float, spent: float, target_path: Path,
paths: PathsTable, visited: List[AgentName],
comp_def: ComputationDef, footprint: float,
replica_count: int, hosts: List[str]):
"""
Visit a path in the replication graph.
Visiting can means attempting to host a replica, if we are on a
__hosting__ node, or forwarding to another agent or answering the
requester.
Parameters
----------
budget
spent
target_path
paths
visited
comp_def
footprint
replica_count
hosts
Returns
-------
forwarded: boolean
a boolean indicating if the request has been answered or
forwarded to another agent.
replica_count: int
the updated replica count.
"""
if target_path.last() == '__hosting__':
# We are actually 'visiting' the '__hosting__' virtual node
# so we must remove it form the paths.
paths.pop(target_path)
if self._can_host(target_path.head(), comp_def.name, footprint):
self._accept_replica(target_path.head(), comp_def, footprint)
hosts.append(self.agent_def.name)
replica_count -= 1
if replica_count == 0:
self.logger.info(
'Target resiliency reached for %s, report back to '
'requester , hosts : %s', comp_def.name, hosts)
self._send_answer(budget, spent, target_path[:-1], paths,
visited, comp_def, footprint,
replica_count, hosts)
return True, replica_count
return False, replica_count
# If the cheapest path was __hosting__, we can still try
# to visit the next path (as we known __hosting__ never
# have any other neighbor) => consider the request as not forwarded
return False, replica_count
self._send_request(budget, spent, target_path, paths, visited,
comp_def, footprint, replica_count, hosts)
return True, replica_count
def _send_request(self, budget: float, spent: float, rq_path: Path,
paths: PathsTable, visited: List[AgentName],
comp_def: ComputationDef, footprint: float,
replica_count: int, hosts: List[AgentName]):
target_agt = rq_path.last()
cost_to_next = self.route(target_agt)
budget_to_next = budget - cost_to_next
spent_to_next = spent + cost_to_next
self.logger.debug(
'sending replica request from %s to %s for %s - %s ('
'budget = %s, cost to next %s)', self.name, target_agt, rq_path,
comp_def.name, budget_to_next, cost_to_next)
self.post_msg(
replication_computation_name(target_agt),
UCSReplicateMessage('replicate_request', budget_to_next,
spent_to_next, rq_path, paths, visited,
comp_def, footprint, replica_count, hosts),
MSG_REPLICATION)
# All request must be answered, otherwise the replication is stuck.
# Keep track of all request sent.
self._pending_requests[(target_agt, comp_def.name)] = \
(budget, spent, rq_path, paths.copy(), visited[:], comp_def,
footprint, replica_count, hosts[:])
def test_filter_missing_agents_paths():
paths = {
Path('_replication_a2', '_replication_a3', '__hosting__'): 4,
Path('_replication_a2', '_replication_a5', '_replication_a6'): 3,
Path('_replication_a3', '_replication_a4'): 1,
Path('_replication_a5', '_replication_a3'): 3,
Path('_replication_a1', '_replication_a4', '_replication_a2'): 3,
}
available = {
'_replication_a2', '_replication_a3', '_replication_a5',
'_replication_a6'
}
filtered = filter_missing_agents_paths(paths, available)
assert len(filtered) == 3
def test_path_iter():
path = Path('a1', 'a2', 'a3')
it = iter(path)
assert 'a1' == next(it)
assert 'a2' == next(it)
assert 'a3' == next(it)
with pytest.raises(StopIteration):
next(it)
def _send_answer(self, budget: float, spent: float, rq_path: Path,
paths: PathsTable, visited: List[AgentName],
comp_def: ComputationDef, footprint: float,
replica_count: int, hosts: List[AgentName]):
assert rq_path.last() == self.agt_name
target_agt = rq_path.before_last()
cost_to_target = self.route(target_agt)
budget += cost_to_target
spent -= cost_to_target
self.logger.debug(
'sending replica answer from %s to %s for %s %s'
'( %s %s %s )', self.name, target_agt, rq_path, comp_def.name,
budget, spent, cost_to_target)
self.post_msg(
replication_computation_name(target_agt),
UCSReplicateMessage('replicate_answer', budget, spent, rq_path,
paths, visited, comp_def, footprint,
replica_count, hosts), MSG_REPLICATION)
def test_path_creation():
assert len(Path(['a', 'b'])) == 2
assert len(Path(['a1'])) == 1
assert len(Path('a1')) == 1
assert len(Path('a', 'b', 'c')) == 3
assert len(Path('a1', 'a2', 'a3')) == 3
assert len(Path(['a1', 'a2'], 'a3')) == 3
assert len(Path()) == 0
def test_path_head():
path = Path('a1', 'a2', 'a3')
assert 'a1' == path.head()
path = Path([])
assert path.head() is None
def test_path_last():
path = Path(['a1', 'a2', 'a3'])
assert 'a3' == path.last()
path = Path([])
assert path.last() is None
def test_paths_starting_with():
paths_starting_a2 = path_starting_with(
Path('_replication_a2', ), {
Path('_replication_a2', '_replication_a3'): 4,
Path('_replication_a2', '_replication_a5', '_replication_a6'): 3,
Path('_replication_a3', '_replication_a4'): 1,
Path('_replication_a4', '_replication_a3'): 3,
})
cost, path = paths_starting_a2[0]
assert cost == 3
assert path == Path('_replication_a5', '_replication_a6')
paths = [path for _, path in paths_starting_a2]
assert Path('_replication_a3', ) in paths
assert Path('_replication_a5', '_replication_a6') in paths
def _add_hosting_path(self, spent: float, computation: ComputationName,
rq_path: Path, paths: PathsTable):
if computation not in self.computations:
# Add a path to a virtual node with a route corresponding to the
# hosting cost
hosting_path = rq_path + Path('__hosting__')
hosting_cost = spent + self.agent_def.hosting_cost(computation)
self.logger.debug(
'Add path to host %s on local hosting node %s '
'with cost %s ',
computation,
hosting_path,
hosting_cost,
)
paths[hosting_path] = hosting_cost
def test_path_add():
p1 = Path('a1')
p2 = Path('a2', 'a3')
assert p1 + p2 == Path('a1', 'a2', 'a3')
assert p2 + p1 == Path('a2', 'a3', 'a1')
p1 = Path()
p2 = Path('a2', 'a3')
assert p1 + p2 == Path('a2', 'a3')
assert p2 + p1 == Path('a2', 'a3')
p1 = Path([])
p2 = Path([])
assert p1 + p2 == Path()
assert p2 + p1 == Path()
def test_path_item():
path = Path('a1', 'a2', 'a3')
assert 'a1' == path[0]
assert Path('a1', 'a2') == path[:-1]
def test_path_empty():
path = Path()
assert path.empty
path = Path('a1', 'a2', 'a3')
assert not path.empty
def test_path_tail_if_start_with():
path = Path(('A', 'B'))
obtained = path.tail_if_start_with(Path('A'))
assert obtained == Path(('B', ))
obtained = path.tail_if_start_with(Path('A', 'B'))
assert obtained == Path(())
obtained = Path('A', 'B', 'C', 'D')\
.tail_if_start_with(Path('A', 'B'))
assert obtained == Path('C', 'D')
obtained = Path('A', 'B', 'C', 'D')\
.tail_if_start_with(Path('A', 'D'))
assert obtained is None
obtained = Path(()).tail_if_start_with(Path('A', 'B'))
assert obtained is None
obtained = Path(('A', 'B', 'C', 'D'))\
.tail_if_start_with(Path())
assert obtained == Path(('A', 'B', 'C', 'D'))
def replicate(self,
k_target: int,
computations: Union[None, ComputationName,
List[ComputationName]] = None):
"""
Launch replication process for the computation(s) passed as argument.
Parameters
----------
k_target: int
target number of replicas
computations: None, string or list of strings.
If computations is a string, is is considered as a single
computation to be replicated/
If it is a list of string, it is the list of computations
that must be registered.
If computations is None, all computations are registered.
"""
if computations is None:
self.logger.info(
'Request for replications of all computations %s -'
' %s', computations, k_target)
computations = [c for c in self.computations]
elif not computations:
self.logger.info('No computation to replicate for %s ', self.name)
self.replication_done(dict(deepcopy(self._replica_hosts)))
return
elif type(computations) == ComputationName:
if computations not in self.computations:
msg = 'Requesting replication of unknown computation {}' \
.format(computations)
self.logger.error(msg)
raise ValueError(msg)
computations = [computations]
else:
unknown = [c for c in computations if c not in self.computations]
if unknown:
msg = 'Requesting replication of unknown computation {}' \
.format(unknown)
self.logger.error(msg)
raise ValueError(msg)
self._replication_in_progress.add(computations)
neighbors = self.replication_neighbors()
if not neighbors:
self.logger.warning(
'Cannot replicate computations %s : no '
'neighbor', computations)
self.replication_done(dict(deepcopy(self._replica_hosts)))
return
self.logger.info(
'Starting replications of computations %s on '
'neighbors %s - %s', computations, neighbors, k_target)
for c in computations:
# initialize paths with our neighbors and their costs
paths = {Path(self.agt_name, n): self.route(n) for n in neighbors}
budget = min(c for c in paths.values())
visited = [self.agt_name]
comp_def, footprint = self.computations[c]
self.on_replicate_request(budget,
0,
Path(self.agt_name),
paths,
visited,
comp_def,
footprint,
replica_count=k_target,
hosts=[])
def test_path_before_last():
path = Path(['a1', 'a2', 'a3'])
assert 'a2' == path.before_last()
path = Path(['a1'])
with pytest.raises(IndexError):
path.before_last()
path = Path([])
with pytest.raises(IndexError):
path.before_last()