def test_compute_ranks_ignore(node_graph): node_to_ignore = set(range(5)) for i in range(5): node_graph.pop(i) visited = graph.compute_ranks(node_graph=node_graph, node_to_ignore=node_to_ignore) for key, rank in visited.items(): assert rank == key - 5
def _add_compute_plan(self, key: str, spec: schemas.ComputePlanSpec, spec_options: dict = None): if spec.clean_models: warnings.warn( "'clean_models=True' is not supported on the local backend.") self._check_metadata(spec.metadata) # Get all the tuples and their dependencies (tuple_graph, traintuples, aggregatetuples, compositetuples) = compute_plan_module.get_dependency_graph(spec) # Define the rank of each traintuple, aggregate tuple and composite tuple visited = graph.compute_ranks(node_graph=tuple_graph) compute_plan = models.ComputePlan(key=key, tag=spec.tag or "", status=models.Status.waiting, metadata=spec.metadata or dict(), id_to_key=dict(), tuple_count=0, done_count=0) compute_plan = self._db.add(compute_plan) # go through the tuples sorted by rank compute_plan = self.__execute_compute_plan(spec, compute_plan, visited, traintuples, aggregatetuples, compositetuples, spec_options) return compute_plan
def test_compute_ranks_no_correlation(): node_graph = { key: list() for key in range(10) } visited = graph.compute_ranks(node_graph=node_graph) for _, rank in visited.items(): assert rank == 0
def update_compute_plan(self, key: str, spec: schemas.UpdateComputePlanSpec, spec_options: dict = None): compute_plan = self._db.get(schemas.Type.ComputePlan, key) # Get all the new tuples and their dependencies (tuple_graph, traintuples, aggregatetuples, compositetuples) = compute_plan_module.get_dependency_graph(spec) # Get the rank of all the tuples already in the compute plan # Tuples already in the CP are identified by both their key and # their id_ in the CP if there is one visited = dict() if compute_plan.traintuple_keys: for key in compute_plan.traintuple_keys: id_, rank = self.__get_id_rank_in_compute_plan( schemas.Type.Traintuple, key, compute_plan.id_to_key, ) visited[key] = rank if id_ is not None: visited[id_] = rank if compute_plan.aggregatetuple_keys: for key in compute_plan.aggregatetuple_keys: id_, rank = self.__get_id_rank_in_compute_plan( schemas.Type.Aggregatetuple, key, compute_plan.id_to_key, ) visited[key] = rank if id_ is not None: visited[id_] = rank if compute_plan.composite_traintuple_keys: for key in compute_plan.composite_traintuple_keys: id_, rank = self.__get_id_rank_in_compute_plan( schemas.Type.CompositeTraintuple, key, compute_plan.id_to_key, ) visited[key] = rank if id_ is not None: visited[id_] = rank # Update the tuple graph with the tuples already in the CP tuple_graph.update({k: list() for k in visited}) visited = graph.compute_ranks(node_graph=tuple_graph, ranks=visited) compute_plan = self.__execute_compute_plan(spec, compute_plan, visited, traintuples, aggregatetuples, compositetuples, spec_options=dict()) return compute_plan
def update_compute_plan(self, key: str, spec: schemas.UpdateComputePlanSpec, spec_options: dict = None): compute_plan = self._db.get(schemas.Type.ComputePlan, key) # Get all the new tuples and their dependencies (tuple_graph, traintuples, aggregatetuples, compositetuples) = compute_plan_module.get_dependency_graph(spec) # Define the rank of each traintuple, aggregate tuple and composite tuple old_tuples = {id_: list() for id_ in compute_plan.id_to_key} tuple_graph.update(old_tuples) # Get the rank of all the tuples already in the compute plan visited = dict() if compute_plan.traintuple_keys: for key in compute_plan.traintuple_keys: id_, rank = self.__get_id_rank_in_compute_plan( schemas.Type.Traintuple, key, compute_plan.id_to_key) visited[id_] = rank if compute_plan.aggregatetuple_keys: for key in compute_plan.aggregatetuple_keys: id_, rank = self.__get_id_rank_in_compute_plan( schemas.Type.Aggregatetuple, key, compute_plan.id_to_key) visited[id_] = rank if compute_plan.composite_traintuple_keys: for key in compute_plan.composite_traintuple_keys: id_, rank = self.__get_id_rank_in_compute_plan( schemas.Type.CompositeTraintuple, key, compute_plan.id_to_key) visited[id_] = rank visited = graph.compute_ranks(node_graph=tuple_graph, ranks=visited) compute_plan = self.__execute_compute_plan(spec, compute_plan, visited, traintuples, aggregatetuples, compositetuples, spec_options=dict()) return compute_plan
def test_compute_ranks_closed_cycle(node_graph_linear): node_graph_linear[0] = [9] with pytest.raises(exceptions.InvalidRequest) as e: graph.compute_ranks(node_graph=node_graph_linear) assert 'missing dependency among inModels IDs' in str(e.value)
def test_compute_ranks_cycle(node_graph): node_graph[5].append(9) with pytest.raises(exceptions.InvalidRequest) as e: graph.compute_ranks(node_graph=node_graph) assert 'missing dependency among inModels IDs' in str(e.value)
def test_compute_ranks_linear(node_graph_linear): visited = graph.compute_ranks(node_graph=node_graph_linear) for key, rank in visited.items(): assert key == rank