def destCity(self, paths: List[List[str]]) -> str:
from graphlib import TopologicalSorter
adict = {}
for key, value in paths:
adict[value] = {key}
ts = TopologicalSorter(adict)
return tuple(ts.static_order())[-1]
def _sort_steps(steps: typing.List["_Step"]) -> typing.Tuple[str, ...]:
"""Return a tuple comprised by step names in the order they should be
executed."""
graph = defaultdict(set)
if len(steps) == 1:
return (steps[0].get_name(), )
edged_nodes = set()
for step in steps:
for other in steps:
if step == other:
continue
step_outputs = set([io.get_name() for io in step.get_outputs()])
other_inputs = set([io.get_name() for io in other.get_inputs()])
if len(step_outputs) > 0 and not step_outputs.isdisjoint(
other_inputs):
graph[other.get_name()].add(step.get_name())
edged_nodes.add(step.get_name())
edged_nodes.add(other.get_name())
isolated_nodes = set([step.get_name() for step in steps]) - edged_nodes
for node in isolated_nodes:
graph[node] = set()
ts = TopologicalSorter(graph)
return tuple(ts.static_order())
def _sort_steps(steps: Sequence["_Step"]) -> Tuple[str, ...]:
"""Return a tuple comprised by step names in the order they should be
executed."""
graph = defaultdict(set)
if len(steps) == 1:
return (steps[0].name, )
edged_nodes = set()
for step in steps:
for other in steps:
if step == other:
continue
step_outputs = set(io.name for io in step.outputs)
other_inputs = set(io.name for io in other.inputs)
if len(step_outputs) > 0 and not step_outputs.isdisjoint(
other_inputs):
graph[other.name].add(step.name)
edged_nodes.add(step.name)
edged_nodes.add(other.name)
isolated_nodes = set(step.name for step in steps) - edged_nodes
for node in isolated_nodes:
graph[node] = set()
ts = TopologicalSorter(graph)
return tuple(ts.static_order())
def find_models(module: ModuleType) -> List[Type[Model]]:
"""
Find all models in a migration script.
"""
models: List[Type[Model]] = []
tables = extract_modified_tables(module)
# add models defined explicitly in the migration script
queue = list(module.__dict__.values())
while queue:
obj = queue.pop()
if hasattr(obj, "__tablename__"):
tables.add(obj.__tablename__)
elif isinstance(obj, list):
queue.extend(obj)
elif isinstance(obj, dict):
queue.extend(obj.values())
# build models by automapping the existing tables, instead of using current
# code; this is needed for migrations that modify schemas (eg, add a column),
# where the current model is out-of-sync with the existing table after a
# downgrade
sqlalchemy_uri = current_app.config["SQLALCHEMY_DATABASE_URI"]
engine = create_engine(sqlalchemy_uri)
Base = automap_base()
Base.prepare(engine, reflect=True)
seen = set()
while tables:
table = tables.pop()
seen.add(table)
model = getattr(Base.classes, table)
model.__tablename__ = table
models.append(model)
# add other models referenced in foreign keys
inspector = inspect(model)
for column in inspector.columns.values():
for foreign_key in column.foreign_keys:
table = foreign_key.column.table.name
if table not in seen:
tables.add(table)
# sort topologically so we can create entities in order and
# maintain relationships (eg, create a database before creating
# a slice)
sorter = TopologicalSorter()
for model in models:
inspector = inspect(model)
dependent_tables: List[str] = []
for column in inspector.columns.values():
for foreign_key in column.foreign_keys:
if foreign_key.column.table.name != model.__tablename__:
dependent_tables.append(foreign_key.column.table.name)
sorter.add(model.__tablename__, *dependent_tables)
order = list(sorter.static_order())
models.sort(key=lambda model: order.index(model.__tablename__))
return models
def get_ready(self):
ts = TopologicalSorter()
nodes = self.get_all().filter(lambda x: x == "not_running").to_dict(
lambda x: x.name)
for node in nodes.values():
depends = Linq(node.depends).filter(
lambda x: not x == "success").map(lambda x: x.name).to_list()
ts.add(node.name, *depends)
return Linq(ts.static_order()).map(lambda x: nodes[x]).save()
def sort(graph: Graph) -> List[Node]:
"""
Linearize a dependency graph, throwing an exception if a cycle is detected.
When no dependencies intervene in ordering, the algorithm preserves the
original insertion order of the graph.
:arg graph: a dict mapping each node to an iterable of their antecedents
"""
sorter = TopologicalSorter(graph)
return list(sorter.static_order())
def _normalize_and_sort_symbols(self):
"""Removes __init__.py from symbols and returns dict sorted by symbol type"""
tmp_sorted = {}
topological_sorter = TopologicalSorter()
relative_star_imports_volume = {}
namespaces = defaultdict(list)
for k, v in sorted(self._original_symbols.items(), key=lambda x: x[1]["type"]):
new_symbol = k.replace(".__init__", "")
if is_relative_import(v):
shadows = resolve_relative_import(
**{k: v for k, v in v["data"].items() if k in {"module", "level", "shadows"}}
)
data = dict(v, data=dict(shadows=shadows))
tmp_sorted[new_symbol] = data
namespace, _, symbol = new_symbol.rpartition(".")
namespaces[namespace].append(new_symbol)
elif is_relative_star_import(v):
imports = [
resolve_relative_import(**{k: v for k, v in data.items() if k in {"module", "level", "shadows"}})
for data in v["data"]["imports"]
]
data = dict(v, data=dict(imports=imports))
topological_sorter.add(new_symbol, *[f"{imp}.{RELATIVE_IMPORT_IDENTIFIER}.*" for imp in imports])
relative_star_imports_volume[new_symbol] = data
namespace, _, symbol = new_symbol.partition(f".{RELATIVE_IMPORT_IDENTIFIER}")
namespaces[namespace].append(new_symbol)
else:
tmp_sorted[new_symbol] = v
if get_symbol_type(v) not in {SymbolType.PACKAGE, SymbolType.MODULE}:
namespace = new_symbol.rpartition(".")[0]
namespaces[namespace].append(new_symbol)
try:
for rel_import in topological_sorter.static_order():
if rel_import in relative_star_imports_volume:
tmp_sorted[rel_import] = relative_star_imports_volume[rel_import]
except CycleError:
pass
shadows_by_in_module_symbol = {
k: v["data"]["shadows"]
for k, v in tmp_sorted.items()
if "shadows" in v["data"] and v["type"] == SymbolType.RELATIVE_IMPORT
}
for k, v in tmp_sorted.items():
volume = v["data"].get("symbols_in_volume", {})
for volume_symbol in list(volume):
no_init_volume_symbol = volume_symbol.replace(".__init__", "")
if no_init_volume_symbol in shadows_by_in_module_symbol:
new_volume_symbol = shadows_by_in_module_symbol[no_init_volume_symbol]
volume[new_volume_symbol] = volume.pop(volume_symbol)
self._sorted_symbols = tmp_sorted
self._namespaces = namespaces
def build(self, step = ''):
dep_graph = TopologicalSorter()
for x, y in self.packages.items():
dep_graph.add(x, *y.requirements)
compile_order = [*dep_graph.static_order()]
packages_to_compile_in_order = []
for x in compile_order:
packages_to_compile_in_order.append(self.packages[x])
if step == 'reinstall':
for pkg in packages_to_compile_in_order:
if (not pkg.install(self.config)):
print(f"Failed to install {pkg.name}!")
return 1
print(f"[nyx]: Installing {pkg.name}-{pkg.version}")
return 0
for pkg in packages_to_compile_in_order:
if (pkg.cached):
print(f"[nyx]: Using cached version of package {pkg.name}-{pkg.version}")
continue
print(f"[nyx]: Compiling {pkg.name}-{pkg.version}")
# Fetch
if (not pkg.fetch(self.config)):
print(f"Failed to fetch {pkg.name}!")
return 1
# Patch
if (not pkg.patch(self.config)):
print(f"Failed to patch {pkg.name}!")
return 1
# Configure
if (not pkg.configure(self.config, self.enviroment)):
print(f"Failed to configure {pkg.name}!")
return 1
# Build
if (not pkg.build(self.config, self.enviroment)):
print(f"Failed to build {pkg.name}!")
return 1
# Package
if (not pkg.package(self.config, self.enviroment)):
print(f"Failed to package {pkg.name}!")
return 1
if (not pkg.install(self.config)):
print(f"Failed to install {pkg.name}!")
return 1
return 0
def init_graph(self, operations_graph):
ts = TopologicalSorter(operations_graph)
elements_queue = [*ts.static_order()]
for v in elements_queue:
self.gather_global_operations(v)
for op_name, op in self.global_operations_info.items():
self.global_operations_info[
op_name] = self.replace_all_get_functions(op)
for v in elements_queue:
(_, element_type, _) = utils.tosca_type_parse(v.type)
if element_type == NODES or element_type == RELATIONSHIPS:
new_conf_args = self.replace_all_get_functions(
v.configuration_args)
v.configuration_args = new_conf_args
else:
del operations_graph[v]
for key in operations_graph:
if v in operations_graph[key]:
operations_graph[key].remove(v)
return operations_graph
def find_models(module: ModuleType) -> List[Type[Model]]:
"""
Find all models in a migration script.
"""
models: List[Type[Model]] = []
tables = extract_modified_tables(module)
# add models defined explicitly in the migration script
queue = list(module.__dict__.values())
while queue:
obj = queue.pop()
if hasattr(obj, "__tablename__"):
tables.add(obj.__tablename__)
elif isinstance(obj, list):
queue.extend(obj)
elif isinstance(obj, dict):
queue.extend(obj.values())
# add implicit models
# pylint: disable=no-member, protected-access
for obj in Model._decl_class_registry.values():
if hasattr(obj, "__table__") and obj.__table__.fullname in tables:
models.append(obj)
# sort topologically so we can create entities in order and
# maintain relationships (eg, create a database before creating
# a slice)
sorter = TopologicalSorter()
for model in models:
inspector = inspect(model)
dependent_tables: List[str] = []
for column in inspector.columns.values():
for foreign_key in column.foreign_keys:
dependent_tables.append(
foreign_key.target_fullname.split(".")[0])
sorter.add(model.__tablename__, *dependent_tables)
order = list(sorter.static_order())
models.sort(key=lambda model: order.index(model.__tablename__))
return models
def topsort(graph):
"""
Topologically sort a graph
The graph should be of the form ``{node: [list of nodes], ...}``.
"""
try:
from graphlib import TopologicalSorter
sorter = TopologicalSorter(graph)
return list(sorter.static_order())
except ImportError:
# TODO: Can be removed when we depend on Python >= 3.9
# make a copy so as not to destroy original
graph = dict((k, copy(v)) for k, v in graph.items())
# Use the standard algorithm for topological sorting:
# http://en.wikipedia.org/wiki/Topological_sorting
# List that will contain the sorted elements
sorted_items = []
# set of all nodes with no incoming edges:
no_incoming = {
node
for node, edges in graph.items() if len(edges) == 0
}
while len(no_incoming):
n = no_incoming.pop()
sorted_items.append(n)
# find nodes m with edges to n
outgoing = [m for m, edges in graph.items() if n in edges]
for m in outgoing:
graph[m].remove(n)
if len(graph[m]) == 0:
# no other dependencies
no_incoming.add(m)
if any([len(edges) > 0 for edges in graph.values()]):
raise ValueError('Cannot topologically sort cyclic graph.')
return sorted_items
"Подготовка VM": [
"Обновить Python до 3.9",
"Клонировать репозиторий курса",
"Установить pyneng.py",
],
"Клонировать репозиторий курса": [
"Подготовка репозитория",
],
"Подготовка репозитория": [
"Исправить ошибки в тестах",
"Исправить ошибки в заданиях",
],
"Исправить ошибки в тестах": [
"Изменить в тестах порядок в assert",
"Вынести общие функции в pyneng_common_functions",
],
"Исправить ошибки в заданиях":
["Исправить ошибки в заданиях из списка todo"],
}
tasks = TopologicalSorter(tasks_dict)
pprint(list(tasks.static_order()))
[
'Обновить Python до 3.9', 'Установить pyneng.py',
'Изменить в тестах порядок в assert',
'Вынести общие функции в pyneng_common_functions',
'Исправить ошибки в заданиях из списка todo', 'Исправить ошибки в тестах',
'Исправить ошибки в заданиях', 'Подготовка репозитория',
'Клонировать репозиторий курса', 'Подготовка VM'
]
traceback.print_exc()
failed = [i.key for i in Status.objects.filter(status='ERROR')]
for i in failed:
recursively_fail(dependency_graph, reversed_dependency_graph, i)
staled = [i.key for i in Status.objects.filter(status='STALED')]
for i in staled:
recursively_skip(dependency_graph, reversed_dependency_graph, i)
building = [i.key for i in Status.objects.filter(status='BUILDING')]
for i in building:
recursively_skip(dependency_graph, reversed_dependency_graph, i)
sorter = TopologicalSorter(dependency_graph)
order = list(sorter.static_order())
order = [i for i in order if i in staled]
resources = config['scheduler']
for group in resources.keys():
if group == 'default':
continue
resources[group]['used'] = Status.objects.filter(
detail__startswith=group).count()
logger.info('%s: %d / %d', group, resources[group]['used'],
resources[group]['total'])
for i in order:
if i == 'dummy':
continue
'myprog': ['extlib1', 'fmtlib', 'urllib', 'mylib'],
'mylib': ['fmtlib', 'stdlib'],
}
do_all_tasks(graph, task_queue, done_queue)
for i in range(NUMBER_OF_PROCESSES):
task_queue.put('STOP')
def do_all_tasks(graph, task_queue, done_queue):
topological_sorter = TopologicalSorter(graph)
topological_sorter.prepare()
while topological_sorter.is_active():
for node in topological_sorter.get_ready():
task_queue.put(node)
node = done_queue.get()
topological_sorter.done(node)
if __name__ == '__main__':
dependencies = {
'scipy': {'numpy'},
'pandas': {'numpy', 'scipy', 'requests'},
'requests': {'urllib3'},
}
from graphlib import TopologicalSorter
ts = TopologicalSorter(dependencies)
print(list(ts.static_order()))
from graphlib import TopologicalSorter
# LIBRARY mapped_to LIBRARY DEPENDENCIES
data = {
'des_system_lib': set('std synopsys std_cell_lib des_system_lib dw02 dw01 ramlib ieee'.split()),
'dw01': set('ieee dw01 dware gtech'.split()),
'dw02': set('ieee dw02 dware'.split()),
'dw03': set('std synopsys dware dw03 dw02 dw01 ieee gtech'.split()),
'dw04': set('dw04 ieee dw01 dware gtech'.split()),
'dw05': set('dw05 ieee dware'.split()),
'dw06': set('dw06 ieee dware'.split()),
'dw07': set('ieee dware'.split()),
'dware': set('ieee dware'.split()),
'gtech': set('ieee gtech'.split()),
'ramlib': set('std ieee'.split()),
'std_cell_lib': set('ieee std_cell_lib'.split()),
'synopsys': set(),
}
# Ignore self dependencies
for k, v in data.items():
v.discard(k)
ts = TopologicalSorter(data)
print(tuple(ts.static_order()))
from graphlib import TopologicalSorter
"""
|- SW11
|- SW2 -|- SW12
SW1 -|
|- SW3 -|- SW13
|- SW14
"""
topology = {
"SW1": ["SW2", "SW3"],
"SW2": ["SW11", "SW12"],
"SW3": ["SW13", "SW14"]
}
top = TopologicalSorter(topology)
print(list(top.static_order()))
# ['SW11', 'SW12', 'SW13', 'SW14', 'SW2', 'SW3', 'SW1']
from graphlib import TopologicalSorter
table_references = {
"customers": set(),
"accounts": {"customers"},
"products": set(),
"orders": {"accounts", "customers"},
"order_products": {"orders", "products"},
}
if __name__ == "__main__":
sorter = TopologicalSorter(table_references)
print(list(sorter.static_order()))
