def destCity(self, paths: List[List[str]]) -> str: from graphlib import TopologicalSorter adict = {} for key, value in paths: adict[value] = {key} ts = TopologicalSorter(adict) return tuple(ts.static_order())[-1]
def _sort_steps(steps: typing.List["_Step"]) -> typing.Tuple[str, ...]: """Return a tuple comprised by step names in the order they should be executed.""" graph = defaultdict(set) if len(steps) == 1: return (steps[0].get_name(), ) edged_nodes = set() for step in steps: for other in steps: if step == other: continue step_outputs = set([io.get_name() for io in step.get_outputs()]) other_inputs = set([io.get_name() for io in other.get_inputs()]) if len(step_outputs) > 0 and not step_outputs.isdisjoint( other_inputs): graph[other.get_name()].add(step.get_name()) edged_nodes.add(step.get_name()) edged_nodes.add(other.get_name()) isolated_nodes = set([step.get_name() for step in steps]) - edged_nodes for node in isolated_nodes: graph[node] = set() ts = TopologicalSorter(graph) return tuple(ts.static_order())
def _sort_steps(steps: Sequence["_Step"]) -> Tuple[str, ...]: """Return a tuple comprised by step names in the order they should be executed.""" graph = defaultdict(set) if len(steps) == 1: return (steps[0].name, ) edged_nodes = set() for step in steps: for other in steps: if step == other: continue step_outputs = set(io.name for io in step.outputs) other_inputs = set(io.name for io in other.inputs) if len(step_outputs) > 0 and not step_outputs.isdisjoint( other_inputs): graph[other.name].add(step.name) edged_nodes.add(step.name) edged_nodes.add(other.name) isolated_nodes = set(step.name for step in steps) - edged_nodes for node in isolated_nodes: graph[node] = set() ts = TopologicalSorter(graph) return tuple(ts.static_order())
def find_models(module: ModuleType) -> List[Type[Model]]: """ Find all models in a migration script. """ models: List[Type[Model]] = [] tables = extract_modified_tables(module) # add models defined explicitly in the migration script queue = list(module.__dict__.values()) while queue: obj = queue.pop() if hasattr(obj, "__tablename__"): tables.add(obj.__tablename__) elif isinstance(obj, list): queue.extend(obj) elif isinstance(obj, dict): queue.extend(obj.values()) # build models by automapping the existing tables, instead of using current # code; this is needed for migrations that modify schemas (eg, add a column), # where the current model is out-of-sync with the existing table after a # downgrade sqlalchemy_uri = current_app.config["SQLALCHEMY_DATABASE_URI"] engine = create_engine(sqlalchemy_uri) Base = automap_base() Base.prepare(engine, reflect=True) seen = set() while tables: table = tables.pop() seen.add(table) model = getattr(Base.classes, table) model.__tablename__ = table models.append(model) # add other models referenced in foreign keys inspector = inspect(model) for column in inspector.columns.values(): for foreign_key in column.foreign_keys: table = foreign_key.column.table.name if table not in seen: tables.add(table) # sort topologically so we can create entities in order and # maintain relationships (eg, create a database before creating # a slice) sorter = TopologicalSorter() for model in models: inspector = inspect(model) dependent_tables: List[str] = [] for column in inspector.columns.values(): for foreign_key in column.foreign_keys: if foreign_key.column.table.name != model.__tablename__: dependent_tables.append(foreign_key.column.table.name) sorter.add(model.__tablename__, *dependent_tables) order = list(sorter.static_order()) models.sort(key=lambda model: order.index(model.__tablename__)) return models
def get_ready(self): ts = TopologicalSorter() nodes = self.get_all().filter(lambda x: x == "not_running").to_dict( lambda x: x.name) for node in nodes.values(): depends = Linq(node.depends).filter( lambda x: not x == "success").map(lambda x: x.name).to_list() ts.add(node.name, *depends) return Linq(ts.static_order()).map(lambda x: nodes[x]).save()
def sort(graph: Graph) -> List[Node]: """ Linearize a dependency graph, throwing an exception if a cycle is detected. When no dependencies intervene in ordering, the algorithm preserves the original insertion order of the graph. :arg graph: a dict mapping each node to an iterable of their antecedents """ sorter = TopologicalSorter(graph) return list(sorter.static_order())
def _normalize_and_sort_symbols(self): """Removes __init__.py from symbols and returns dict sorted by symbol type""" tmp_sorted = {} topological_sorter = TopologicalSorter() relative_star_imports_volume = {} namespaces = defaultdict(list) for k, v in sorted(self._original_symbols.items(), key=lambda x: x[1]["type"]): new_symbol = k.replace(".__init__", "") if is_relative_import(v): shadows = resolve_relative_import( **{k: v for k, v in v["data"].items() if k in {"module", "level", "shadows"}} ) data = dict(v, data=dict(shadows=shadows)) tmp_sorted[new_symbol] = data namespace, _, symbol = new_symbol.rpartition(".") namespaces[namespace].append(new_symbol) elif is_relative_star_import(v): imports = [ resolve_relative_import(**{k: v for k, v in data.items() if k in {"module", "level", "shadows"}}) for data in v["data"]["imports"] ] data = dict(v, data=dict(imports=imports)) topological_sorter.add(new_symbol, *[f"{imp}.{RELATIVE_IMPORT_IDENTIFIER}.*" for imp in imports]) relative_star_imports_volume[new_symbol] = data namespace, _, symbol = new_symbol.partition(f".{RELATIVE_IMPORT_IDENTIFIER}") namespaces[namespace].append(new_symbol) else: tmp_sorted[new_symbol] = v if get_symbol_type(v) not in {SymbolType.PACKAGE, SymbolType.MODULE}: namespace = new_symbol.rpartition(".")[0] namespaces[namespace].append(new_symbol) try: for rel_import in topological_sorter.static_order(): if rel_import in relative_star_imports_volume: tmp_sorted[rel_import] = relative_star_imports_volume[rel_import] except CycleError: pass shadows_by_in_module_symbol = { k: v["data"]["shadows"] for k, v in tmp_sorted.items() if "shadows" in v["data"] and v["type"] == SymbolType.RELATIVE_IMPORT } for k, v in tmp_sorted.items(): volume = v["data"].get("symbols_in_volume", {}) for volume_symbol in list(volume): no_init_volume_symbol = volume_symbol.replace(".__init__", "") if no_init_volume_symbol in shadows_by_in_module_symbol: new_volume_symbol = shadows_by_in_module_symbol[no_init_volume_symbol] volume[new_volume_symbol] = volume.pop(volume_symbol) self._sorted_symbols = tmp_sorted self._namespaces = namespaces
def build(self, step = ''): dep_graph = TopologicalSorter() for x, y in self.packages.items(): dep_graph.add(x, *y.requirements) compile_order = [*dep_graph.static_order()] packages_to_compile_in_order = [] for x in compile_order: packages_to_compile_in_order.append(self.packages[x]) if step == 'reinstall': for pkg in packages_to_compile_in_order: if (not pkg.install(self.config)): print(f"Failed to install {pkg.name}!") return 1 print(f"[nyx]: Installing {pkg.name}-{pkg.version}") return 0 for pkg in packages_to_compile_in_order: if (pkg.cached): print(f"[nyx]: Using cached version of package {pkg.name}-{pkg.version}") continue print(f"[nyx]: Compiling {pkg.name}-{pkg.version}") # Fetch if (not pkg.fetch(self.config)): print(f"Failed to fetch {pkg.name}!") return 1 # Patch if (not pkg.patch(self.config)): print(f"Failed to patch {pkg.name}!") return 1 # Configure if (not pkg.configure(self.config, self.enviroment)): print(f"Failed to configure {pkg.name}!") return 1 # Build if (not pkg.build(self.config, self.enviroment)): print(f"Failed to build {pkg.name}!") return 1 # Package if (not pkg.package(self.config, self.enviroment)): print(f"Failed to package {pkg.name}!") return 1 if (not pkg.install(self.config)): print(f"Failed to install {pkg.name}!") return 1 return 0
def init_graph(self, operations_graph): ts = TopologicalSorter(operations_graph) elements_queue = [*ts.static_order()] for v in elements_queue: self.gather_global_operations(v) for op_name, op in self.global_operations_info.items(): self.global_operations_info[ op_name] = self.replace_all_get_functions(op) for v in elements_queue: (_, element_type, _) = utils.tosca_type_parse(v.type) if element_type == NODES or element_type == RELATIONSHIPS: new_conf_args = self.replace_all_get_functions( v.configuration_args) v.configuration_args = new_conf_args else: del operations_graph[v] for key in operations_graph: if v in operations_graph[key]: operations_graph[key].remove(v) return operations_graph
def find_models(module: ModuleType) -> List[Type[Model]]: """ Find all models in a migration script. """ models: List[Type[Model]] = [] tables = extract_modified_tables(module) # add models defined explicitly in the migration script queue = list(module.__dict__.values()) while queue: obj = queue.pop() if hasattr(obj, "__tablename__"): tables.add(obj.__tablename__) elif isinstance(obj, list): queue.extend(obj) elif isinstance(obj, dict): queue.extend(obj.values()) # add implicit models # pylint: disable=no-member, protected-access for obj in Model._decl_class_registry.values(): if hasattr(obj, "__table__") and obj.__table__.fullname in tables: models.append(obj) # sort topologically so we can create entities in order and # maintain relationships (eg, create a database before creating # a slice) sorter = TopologicalSorter() for model in models: inspector = inspect(model) dependent_tables: List[str] = [] for column in inspector.columns.values(): for foreign_key in column.foreign_keys: dependent_tables.append( foreign_key.target_fullname.split(".")[0]) sorter.add(model.__tablename__, *dependent_tables) order = list(sorter.static_order()) models.sort(key=lambda model: order.index(model.__tablename__)) return models
def topsort(graph): """ Topologically sort a graph The graph should be of the form ``{node: [list of nodes], ...}``. """ try: from graphlib import TopologicalSorter sorter = TopologicalSorter(graph) return list(sorter.static_order()) except ImportError: # TODO: Can be removed when we depend on Python >= 3.9 # make a copy so as not to destroy original graph = dict((k, copy(v)) for k, v in graph.items()) # Use the standard algorithm for topological sorting: # http://en.wikipedia.org/wiki/Topological_sorting # List that will contain the sorted elements sorted_items = [] # set of all nodes with no incoming edges: no_incoming = { node for node, edges in graph.items() if len(edges) == 0 } while len(no_incoming): n = no_incoming.pop() sorted_items.append(n) # find nodes m with edges to n outgoing = [m for m, edges in graph.items() if n in edges] for m in outgoing: graph[m].remove(n) if len(graph[m]) == 0: # no other dependencies no_incoming.add(m) if any([len(edges) > 0 for edges in graph.values()]): raise ValueError('Cannot topologically sort cyclic graph.') return sorted_items
"Подготовка VM": [ "Обновить Python до 3.9", "Клонировать репозиторий курса", "Установить pyneng.py", ], "Клонировать репозиторий курса": [ "Подготовка репозитория", ], "Подготовка репозитория": [ "Исправить ошибки в тестах", "Исправить ошибки в заданиях", ], "Исправить ошибки в тестах": [ "Изменить в тестах порядок в assert", "Вынести общие функции в pyneng_common_functions", ], "Исправить ошибки в заданиях": ["Исправить ошибки в заданиях из списка todo"], } tasks = TopologicalSorter(tasks_dict) pprint(list(tasks.static_order())) [ 'Обновить Python до 3.9', 'Установить pyneng.py', 'Изменить в тестах порядок в assert', 'Вынести общие функции в pyneng_common_functions', 'Исправить ошибки в заданиях из списка todo', 'Исправить ошибки в тестах', 'Исправить ошибки в заданиях', 'Подготовка репозитория', 'Клонировать репозиторий курса', 'Подготовка VM' ]
traceback.print_exc() failed = [i.key for i in Status.objects.filter(status='ERROR')] for i in failed: recursively_fail(dependency_graph, reversed_dependency_graph, i) staled = [i.key for i in Status.objects.filter(status='STALED')] for i in staled: recursively_skip(dependency_graph, reversed_dependency_graph, i) building = [i.key for i in Status.objects.filter(status='BUILDING')] for i in building: recursively_skip(dependency_graph, reversed_dependency_graph, i) sorter = TopologicalSorter(dependency_graph) order = list(sorter.static_order()) order = [i for i in order if i in staled] resources = config['scheduler'] for group in resources.keys(): if group == 'default': continue resources[group]['used'] = Status.objects.filter( detail__startswith=group).count() logger.info('%s: %d / %d', group, resources[group]['used'], resources[group]['total']) for i in order: if i == 'dummy': continue
'myprog': ['extlib1', 'fmtlib', 'urllib', 'mylib'], 'mylib': ['fmtlib', 'stdlib'], } do_all_tasks(graph, task_queue, done_queue) for i in range(NUMBER_OF_PROCESSES): task_queue.put('STOP') def do_all_tasks(graph, task_queue, done_queue): topological_sorter = TopologicalSorter(graph) topological_sorter.prepare() while topological_sorter.is_active(): for node in topological_sorter.get_ready(): task_queue.put(node) node = done_queue.get() topological_sorter.done(node) if __name__ == '__main__': dependencies = { 'scipy': {'numpy'}, 'pandas': {'numpy', 'scipy', 'requests'}, 'requests': {'urllib3'}, } from graphlib import TopologicalSorter ts = TopologicalSorter(dependencies) print(list(ts.static_order()))
from graphlib import TopologicalSorter # LIBRARY mapped_to LIBRARY DEPENDENCIES data = { 'des_system_lib': set('std synopsys std_cell_lib des_system_lib dw02 dw01 ramlib ieee'.split()), 'dw01': set('ieee dw01 dware gtech'.split()), 'dw02': set('ieee dw02 dware'.split()), 'dw03': set('std synopsys dware dw03 dw02 dw01 ieee gtech'.split()), 'dw04': set('dw04 ieee dw01 dware gtech'.split()), 'dw05': set('dw05 ieee dware'.split()), 'dw06': set('dw06 ieee dware'.split()), 'dw07': set('ieee dware'.split()), 'dware': set('ieee dware'.split()), 'gtech': set('ieee gtech'.split()), 'ramlib': set('std ieee'.split()), 'std_cell_lib': set('ieee std_cell_lib'.split()), 'synopsys': set(), } # Ignore self dependencies for k, v in data.items(): v.discard(k) ts = TopologicalSorter(data) print(tuple(ts.static_order()))
from graphlib import TopologicalSorter """ |- SW11 |- SW2 -|- SW12 SW1 -| |- SW3 -|- SW13 |- SW14 """ topology = { "SW1": ["SW2", "SW3"], "SW2": ["SW11", "SW12"], "SW3": ["SW13", "SW14"] } top = TopologicalSorter(topology) print(list(top.static_order())) # ['SW11', 'SW12', 'SW13', 'SW14', 'SW2', 'SW3', 'SW1']
from graphlib import TopologicalSorter table_references = { "customers": set(), "accounts": {"customers"}, "products": set(), "orders": {"accounts", "customers"}, "order_products": {"orders", "products"}, } if __name__ == "__main__": sorter = TopologicalSorter(table_references) print(list(sorter.static_order()))