def get_op_order(self):
ts = TopologicalSorter()
for host in self.inventory:
for i, op_hash in enumerate(host.op_hash_order):
if not i:
ts.add(op_hash)
else:
ts.add(op_hash, host.op_hash_order[i - 1])
final_op_order = []
ts.prepare()
while ts.is_active():
# Ensure that where we have multiple different operations that can be executed in any
# dependency order we order them by line numbers.
node_group = sorted(
ts.get_ready(),
key=lambda op_hash: self.op_meta[op_hash]["op_order"],
)
ts.done(*node_group)
final_op_order.extend(node_group)
return final_op_order
def find_models(module: ModuleType) -> List[Type[Model]]:
"""
Find all models in a migration script.
"""
models: List[Type[Model]] = []
tables = extract_modified_tables(module)
# add models defined explicitly in the migration script
queue = list(module.__dict__.values())
while queue:
obj = queue.pop()
if hasattr(obj, "__tablename__"):
tables.add(obj.__tablename__)
elif isinstance(obj, list):
queue.extend(obj)
elif isinstance(obj, dict):
queue.extend(obj.values())
# build models by automapping the existing tables, instead of using current
# code; this is needed for migrations that modify schemas (eg, add a column),
# where the current model is out-of-sync with the existing table after a
# downgrade
sqlalchemy_uri = current_app.config["SQLALCHEMY_DATABASE_URI"]
engine = create_engine(sqlalchemy_uri)
Base = automap_base()
Base.prepare(engine, reflect=True)
seen = set()
while tables:
table = tables.pop()
seen.add(table)
model = getattr(Base.classes, table)
model.__tablename__ = table
models.append(model)
# add other models referenced in foreign keys
inspector = inspect(model)
for column in inspector.columns.values():
for foreign_key in column.foreign_keys:
table = foreign_key.column.table.name
if table not in seen:
tables.add(table)
# sort topologically so we can create entities in order and
# maintain relationships (eg, create a database before creating
# a slice)
sorter = TopologicalSorter()
for model in models:
inspector = inspect(model)
dependent_tables: List[str] = []
for column in inspector.columns.values():
for foreign_key in column.foreign_keys:
if foreign_key.column.table.name != model.__tablename__:
dependent_tables.append(foreign_key.column.table.name)
sorter.add(model.__tablename__, *dependent_tables)
order = list(sorter.static_order())
models.sort(key=lambda model: order.index(model.__tablename__))
return models
def get_ready(self):
ts = TopologicalSorter()
nodes = self.get_all().filter(lambda x: x == "not_running").to_dict(
lambda x: x.name)
for node in nodes.values():
depends = Linq(node.depends).filter(
lambda x: not x == "success").map(lambda x: x.name).to_list()
ts.add(node.name, *depends)
return Linq(ts.static_order()).map(lambda x: nodes[x]).save()
def validate(self, result: Announcement):
graph = TopologicalSorter()
for dep in result.dependencies:
graph.add(dep)
for key, message in self.items():
for dep in message.dependencies:
graph.add(dep, key)
graph.prepare()
missing = graph._node2info.keys() - self.keys()
if missing:
raise ValueError(f"Missing items: {','.join(missing)}")
def _normalize_and_sort_symbols(self):
"""Removes __init__.py from symbols and returns dict sorted by symbol type"""
tmp_sorted = {}
topological_sorter = TopologicalSorter()
relative_star_imports_volume = {}
namespaces = defaultdict(list)
for k, v in sorted(self._original_symbols.items(), key=lambda x: x[1]["type"]):
new_symbol = k.replace(".__init__", "")
if is_relative_import(v):
shadows = resolve_relative_import(
**{k: v for k, v in v["data"].items() if k in {"module", "level", "shadows"}}
)
data = dict(v, data=dict(shadows=shadows))
tmp_sorted[new_symbol] = data
namespace, _, symbol = new_symbol.rpartition(".")
namespaces[namespace].append(new_symbol)
elif is_relative_star_import(v):
imports = [
resolve_relative_import(**{k: v for k, v in data.items() if k in {"module", "level", "shadows"}})
for data in v["data"]["imports"]
]
data = dict(v, data=dict(imports=imports))
topological_sorter.add(new_symbol, *[f"{imp}.{RELATIVE_IMPORT_IDENTIFIER}.*" for imp in imports])
relative_star_imports_volume[new_symbol] = data
namespace, _, symbol = new_symbol.partition(f".{RELATIVE_IMPORT_IDENTIFIER}")
namespaces[namespace].append(new_symbol)
else:
tmp_sorted[new_symbol] = v
if get_symbol_type(v) not in {SymbolType.PACKAGE, SymbolType.MODULE}:
namespace = new_symbol.rpartition(".")[0]
namespaces[namespace].append(new_symbol)
try:
for rel_import in topological_sorter.static_order():
if rel_import in relative_star_imports_volume:
tmp_sorted[rel_import] = relative_star_imports_volume[rel_import]
except CycleError:
pass
shadows_by_in_module_symbol = {
k: v["data"]["shadows"]
for k, v in tmp_sorted.items()
if "shadows" in v["data"] and v["type"] == SymbolType.RELATIVE_IMPORT
}
for k, v in tmp_sorted.items():
volume = v["data"].get("symbols_in_volume", {})
for volume_symbol in list(volume):
no_init_volume_symbol = volume_symbol.replace(".__init__", "")
if no_init_volume_symbol in shadows_by_in_module_symbol:
new_volume_symbol = shadows_by_in_module_symbol[no_init_volume_symbol]
volume[new_volume_symbol] = volume.pop(volume_symbol)
self._sorted_symbols = tmp_sorted
self._namespaces = namespaces
def build(self, step = ''):
dep_graph = TopologicalSorter()
for x, y in self.packages.items():
dep_graph.add(x, *y.requirements)
compile_order = [*dep_graph.static_order()]
packages_to_compile_in_order = []
for x in compile_order:
packages_to_compile_in_order.append(self.packages[x])
if step == 'reinstall':
for pkg in packages_to_compile_in_order:
if (not pkg.install(self.config)):
print(f"Failed to install {pkg.name}!")
return 1
print(f"[nyx]: Installing {pkg.name}-{pkg.version}")
return 0
for pkg in packages_to_compile_in_order:
if (pkg.cached):
print(f"[nyx]: Using cached version of package {pkg.name}-{pkg.version}")
continue
print(f"[nyx]: Compiling {pkg.name}-{pkg.version}")
# Fetch
if (not pkg.fetch(self.config)):
print(f"Failed to fetch {pkg.name}!")
return 1
# Patch
if (not pkg.patch(self.config)):
print(f"Failed to patch {pkg.name}!")
return 1
# Configure
if (not pkg.configure(self.config, self.enviroment)):
print(f"Failed to configure {pkg.name}!")
return 1
# Build
if (not pkg.build(self.config, self.enviroment)):
print(f"Failed to build {pkg.name}!")
return 1
# Package
if (not pkg.package(self.config, self.enviroment)):
print(f"Failed to package {pkg.name}!")
return 1
if (not pkg.install(self.config)):
print(f"Failed to install {pkg.name}!")
return 1
return 0
def find_models(module: ModuleType) -> List[Type[Model]]:
"""
Find all models in a migration script.
"""
models: List[Type[Model]] = []
tables = extract_modified_tables(module)
# add models defined explicitly in the migration script
queue = list(module.__dict__.values())
while queue:
obj = queue.pop()
if hasattr(obj, "__tablename__"):
tables.add(obj.__tablename__)
elif isinstance(obj, list):
queue.extend(obj)
elif isinstance(obj, dict):
queue.extend(obj.values())
# add implicit models
# pylint: disable=no-member, protected-access
for obj in Model._decl_class_registry.values():
if hasattr(obj, "__table__") and obj.__table__.fullname in tables:
models.append(obj)
# sort topologically so we can create entities in order and
# maintain relationships (eg, create a database before creating
# a slice)
sorter = TopologicalSorter()
for model in models:
inspector = inspect(model)
dependent_tables: List[str] = []
for column in inspector.columns.values():
for foreign_key in column.foreign_keys:
dependent_tables.append(
foreign_key.target_fullname.split(".")[0])
sorter.add(model.__tablename__, *dependent_tables)
order = list(sorter.static_order())
models.sort(key=lambda model: order.index(model.__tablename__))
return models
import fileinput as fi
import heapq
from graphlib import TopologicalSorter
ts = TopologicalSorter()
for line in fi.input():
pr = line.split(" ")
a, b = pr[1], pr[-3]
ts.add(b, a)
ts.prepare()
EXTRA = 60
# Worker heaps
IDLE = 5
BUSY = []
Q = list(ts.get_ready())
heapq.heapify(Q)
t_min = 0
while ts.is_active():
# While there are tasks and idle workers, we hand out tasks.
while Q and IDLE > 0:
c = heapq.heappop(Q)
IDLE -= 1
heapq.heappush(BUSY, (t_min + ord(c) - (ord("A") - 1) + EXTRA, c))
