def test_compute_ranks_ignore(node_graph):
node_to_ignore = set(range(5))
for i in range(5):
node_graph.pop(i)
visited = graph.compute_ranks(node_graph=node_graph, node_to_ignore=node_to_ignore)
for key, rank in visited.items():
assert rank == key - 5
def _add_compute_plan(self,
key: str,
spec: schemas.ComputePlanSpec,
spec_options: dict = None):
if spec.clean_models:
warnings.warn(
"'clean_models=True' is not supported on the local backend.")
self._check_metadata(spec.metadata)
# Get all the tuples and their dependencies
(tuple_graph, traintuples, aggregatetuples,
compositetuples) = compute_plan_module.get_dependency_graph(spec)
# Define the rank of each traintuple, aggregate tuple and composite tuple
visited = graph.compute_ranks(node_graph=tuple_graph)
compute_plan = models.ComputePlan(key=key,
tag=spec.tag or "",
status=models.Status.waiting,
metadata=spec.metadata or dict(),
id_to_key=dict(),
tuple_count=0,
done_count=0)
compute_plan = self._db.add(compute_plan)
# go through the tuples sorted by rank
compute_plan = self.__execute_compute_plan(spec, compute_plan, visited,
traintuples,
aggregatetuples,
compositetuples,
spec_options)
return compute_plan
def test_compute_ranks_no_correlation():
node_graph = {
key: list()
for key in range(10)
}
visited = graph.compute_ranks(node_graph=node_graph)
for _, rank in visited.items():
assert rank == 0
def update_compute_plan(self,
key: str,
spec: schemas.UpdateComputePlanSpec,
spec_options: dict = None):
compute_plan = self._db.get(schemas.Type.ComputePlan, key)
# Get all the new tuples and their dependencies
(tuple_graph, traintuples, aggregatetuples,
compositetuples) = compute_plan_module.get_dependency_graph(spec)
# Get the rank of all the tuples already in the compute plan
# Tuples already in the CP are identified by both their key and
# their id_ in the CP if there is one
visited = dict()
if compute_plan.traintuple_keys:
for key in compute_plan.traintuple_keys:
id_, rank = self.__get_id_rank_in_compute_plan(
schemas.Type.Traintuple,
key,
compute_plan.id_to_key,
)
visited[key] = rank
if id_ is not None:
visited[id_] = rank
if compute_plan.aggregatetuple_keys:
for key in compute_plan.aggregatetuple_keys:
id_, rank = self.__get_id_rank_in_compute_plan(
schemas.Type.Aggregatetuple,
key,
compute_plan.id_to_key,
)
visited[key] = rank
if id_ is not None:
visited[id_] = rank
if compute_plan.composite_traintuple_keys:
for key in compute_plan.composite_traintuple_keys:
id_, rank = self.__get_id_rank_in_compute_plan(
schemas.Type.CompositeTraintuple,
key,
compute_plan.id_to_key,
)
visited[key] = rank
if id_ is not None:
visited[id_] = rank
# Update the tuple graph with the tuples already in the CP
tuple_graph.update({k: list() for k in visited})
visited = graph.compute_ranks(node_graph=tuple_graph, ranks=visited)
compute_plan = self.__execute_compute_plan(spec,
compute_plan,
visited,
traintuples,
aggregatetuples,
compositetuples,
spec_options=dict())
return compute_plan
def update_compute_plan(self,
key: str,
spec: schemas.UpdateComputePlanSpec,
spec_options: dict = None):
compute_plan = self._db.get(schemas.Type.ComputePlan, key)
# Get all the new tuples and their dependencies
(tuple_graph, traintuples, aggregatetuples,
compositetuples) = compute_plan_module.get_dependency_graph(spec)
# Define the rank of each traintuple, aggregate tuple and composite tuple
old_tuples = {id_: list() for id_ in compute_plan.id_to_key}
tuple_graph.update(old_tuples)
# Get the rank of all the tuples already in the compute plan
visited = dict()
if compute_plan.traintuple_keys:
for key in compute_plan.traintuple_keys:
id_, rank = self.__get_id_rank_in_compute_plan(
schemas.Type.Traintuple, key, compute_plan.id_to_key)
visited[id_] = rank
if compute_plan.aggregatetuple_keys:
for key in compute_plan.aggregatetuple_keys:
id_, rank = self.__get_id_rank_in_compute_plan(
schemas.Type.Aggregatetuple, key, compute_plan.id_to_key)
visited[id_] = rank
if compute_plan.composite_traintuple_keys:
for key in compute_plan.composite_traintuple_keys:
id_, rank = self.__get_id_rank_in_compute_plan(
schemas.Type.CompositeTraintuple, key,
compute_plan.id_to_key)
visited[id_] = rank
visited = graph.compute_ranks(node_graph=tuple_graph, ranks=visited)
compute_plan = self.__execute_compute_plan(spec,
compute_plan,
visited,
traintuples,
aggregatetuples,
compositetuples,
spec_options=dict())
return compute_plan
def test_compute_ranks_closed_cycle(node_graph_linear):
node_graph_linear[0] = [9]
with pytest.raises(exceptions.InvalidRequest) as e:
graph.compute_ranks(node_graph=node_graph_linear)
assert 'missing dependency among inModels IDs' in str(e.value)
def test_compute_ranks_cycle(node_graph):
node_graph[5].append(9)
with pytest.raises(exceptions.InvalidRequest) as e:
graph.compute_ranks(node_graph=node_graph)
assert 'missing dependency among inModels IDs' in str(e.value)
def test_compute_ranks_linear(node_graph_linear):
visited = graph.compute_ranks(node_graph=node_graph_linear)
for key, rank in visited.items():
assert key == rank
