def get_op_order(self): ts = TopologicalSorter() for host in self.inventory: for i, op_hash in enumerate(host.op_hash_order): if not i: ts.add(op_hash) else: ts.add(op_hash, host.op_hash_order[i - 1]) final_op_order = [] ts.prepare() while ts.is_active(): # Ensure that where we have multiple different operations that can be executed in any # dependency order we order them by line numbers. node_group = sorted( ts.get_ready(), key=lambda op_hash: self.op_meta[op_hash]["op_order"], ) ts.done(*node_group) final_op_order.extend(node_group) return final_op_order
def find_models(module: ModuleType) -> List[Type[Model]]: """ Find all models in a migration script. """ models: List[Type[Model]] = [] tables = extract_modified_tables(module) # add models defined explicitly in the migration script queue = list(module.__dict__.values()) while queue: obj = queue.pop() if hasattr(obj, "__tablename__"): tables.add(obj.__tablename__) elif isinstance(obj, list): queue.extend(obj) elif isinstance(obj, dict): queue.extend(obj.values()) # build models by automapping the existing tables, instead of using current # code; this is needed for migrations that modify schemas (eg, add a column), # where the current model is out-of-sync with the existing table after a # downgrade sqlalchemy_uri = current_app.config["SQLALCHEMY_DATABASE_URI"] engine = create_engine(sqlalchemy_uri) Base = automap_base() Base.prepare(engine, reflect=True) seen = set() while tables: table = tables.pop() seen.add(table) model = getattr(Base.classes, table) model.__tablename__ = table models.append(model) # add other models referenced in foreign keys inspector = inspect(model) for column in inspector.columns.values(): for foreign_key in column.foreign_keys: table = foreign_key.column.table.name if table not in seen: tables.add(table) # sort topologically so we can create entities in order and # maintain relationships (eg, create a database before creating # a slice) sorter = TopologicalSorter() for model in models: inspector = inspect(model) dependent_tables: List[str] = [] for column in inspector.columns.values(): for foreign_key in column.foreign_keys: if foreign_key.column.table.name != model.__tablename__: dependent_tables.append(foreign_key.column.table.name) sorter.add(model.__tablename__, *dependent_tables) order = list(sorter.static_order()) models.sort(key=lambda model: order.index(model.__tablename__)) return models
def get_ready(self): ts = TopologicalSorter() nodes = self.get_all().filter(lambda x: x == "not_running").to_dict( lambda x: x.name) for node in nodes.values(): depends = Linq(node.depends).filter( lambda x: not x == "success").map(lambda x: x.name).to_list() ts.add(node.name, *depends) return Linq(ts.static_order()).map(lambda x: nodes[x]).save()
def validate(self, result: Announcement): graph = TopologicalSorter() for dep in result.dependencies: graph.add(dep) for key, message in self.items(): for dep in message.dependencies: graph.add(dep, key) graph.prepare() missing = graph._node2info.keys() - self.keys() if missing: raise ValueError(f"Missing items: {','.join(missing)}")
def _normalize_and_sort_symbols(self): """Removes __init__.py from symbols and returns dict sorted by symbol type""" tmp_sorted = {} topological_sorter = TopologicalSorter() relative_star_imports_volume = {} namespaces = defaultdict(list) for k, v in sorted(self._original_symbols.items(), key=lambda x: x[1]["type"]): new_symbol = k.replace(".__init__", "") if is_relative_import(v): shadows = resolve_relative_import( **{k: v for k, v in v["data"].items() if k in {"module", "level", "shadows"}} ) data = dict(v, data=dict(shadows=shadows)) tmp_sorted[new_symbol] = data namespace, _, symbol = new_symbol.rpartition(".") namespaces[namespace].append(new_symbol) elif is_relative_star_import(v): imports = [ resolve_relative_import(**{k: v for k, v in data.items() if k in {"module", "level", "shadows"}}) for data in v["data"]["imports"] ] data = dict(v, data=dict(imports=imports)) topological_sorter.add(new_symbol, *[f"{imp}.{RELATIVE_IMPORT_IDENTIFIER}.*" for imp in imports]) relative_star_imports_volume[new_symbol] = data namespace, _, symbol = new_symbol.partition(f".{RELATIVE_IMPORT_IDENTIFIER}") namespaces[namespace].append(new_symbol) else: tmp_sorted[new_symbol] = v if get_symbol_type(v) not in {SymbolType.PACKAGE, SymbolType.MODULE}: namespace = new_symbol.rpartition(".")[0] namespaces[namespace].append(new_symbol) try: for rel_import in topological_sorter.static_order(): if rel_import in relative_star_imports_volume: tmp_sorted[rel_import] = relative_star_imports_volume[rel_import] except CycleError: pass shadows_by_in_module_symbol = { k: v["data"]["shadows"] for k, v in tmp_sorted.items() if "shadows" in v["data"] and v["type"] == SymbolType.RELATIVE_IMPORT } for k, v in tmp_sorted.items(): volume = v["data"].get("symbols_in_volume", {}) for volume_symbol in list(volume): no_init_volume_symbol = volume_symbol.replace(".__init__", "") if no_init_volume_symbol in shadows_by_in_module_symbol: new_volume_symbol = shadows_by_in_module_symbol[no_init_volume_symbol] volume[new_volume_symbol] = volume.pop(volume_symbol) self._sorted_symbols = tmp_sorted self._namespaces = namespaces
def build(self, step = ''): dep_graph = TopologicalSorter() for x, y in self.packages.items(): dep_graph.add(x, *y.requirements) compile_order = [*dep_graph.static_order()] packages_to_compile_in_order = [] for x in compile_order: packages_to_compile_in_order.append(self.packages[x]) if step == 'reinstall': for pkg in packages_to_compile_in_order: if (not pkg.install(self.config)): print(f"Failed to install {pkg.name}!") return 1 print(f"[nyx]: Installing {pkg.name}-{pkg.version}") return 0 for pkg in packages_to_compile_in_order: if (pkg.cached): print(f"[nyx]: Using cached version of package {pkg.name}-{pkg.version}") continue print(f"[nyx]: Compiling {pkg.name}-{pkg.version}") # Fetch if (not pkg.fetch(self.config)): print(f"Failed to fetch {pkg.name}!") return 1 # Patch if (not pkg.patch(self.config)): print(f"Failed to patch {pkg.name}!") return 1 # Configure if (not pkg.configure(self.config, self.enviroment)): print(f"Failed to configure {pkg.name}!") return 1 # Build if (not pkg.build(self.config, self.enviroment)): print(f"Failed to build {pkg.name}!") return 1 # Package if (not pkg.package(self.config, self.enviroment)): print(f"Failed to package {pkg.name}!") return 1 if (not pkg.install(self.config)): print(f"Failed to install {pkg.name}!") return 1 return 0
def find_models(module: ModuleType) -> List[Type[Model]]: """ Find all models in a migration script. """ models: List[Type[Model]] = [] tables = extract_modified_tables(module) # add models defined explicitly in the migration script queue = list(module.__dict__.values()) while queue: obj = queue.pop() if hasattr(obj, "__tablename__"): tables.add(obj.__tablename__) elif isinstance(obj, list): queue.extend(obj) elif isinstance(obj, dict): queue.extend(obj.values()) # add implicit models # pylint: disable=no-member, protected-access for obj in Model._decl_class_registry.values(): if hasattr(obj, "__table__") and obj.__table__.fullname in tables: models.append(obj) # sort topologically so we can create entities in order and # maintain relationships (eg, create a database before creating # a slice) sorter = TopologicalSorter() for model in models: inspector = inspect(model) dependent_tables: List[str] = [] for column in inspector.columns.values(): for foreign_key in column.foreign_keys: dependent_tables.append( foreign_key.target_fullname.split(".")[0]) sorter.add(model.__tablename__, *dependent_tables) order = list(sorter.static_order()) models.sort(key=lambda model: order.index(model.__tablename__)) return models
import fileinput as fi import heapq from graphlib import TopologicalSorter ts = TopologicalSorter() for line in fi.input(): pr = line.split(" ") a, b = pr[1], pr[-3] ts.add(b, a) ts.prepare() EXTRA = 60 # Worker heaps IDLE = 5 BUSY = [] Q = list(ts.get_ready()) heapq.heapify(Q) t_min = 0 while ts.is_active(): # While there are tasks and idle workers, we hand out tasks. while Q and IDLE > 0: c = heapq.heappop(Q) IDLE -= 1 heapq.heappush(BUSY, (t_min + ord(c) - (ord("A") - 1) + EXTRA, c))