def _sort_steps(steps: typing.List["_Step"]) -> typing.Tuple[str, ...]: """Return a tuple comprised by step names in the order they should be executed.""" graph = defaultdict(set) if len(steps) == 1: return (steps[0].get_name(), ) edged_nodes = set() for step in steps: for other in steps: if step == other: continue step_outputs = set([io.get_name() for io in step.get_outputs()]) other_inputs = set([io.get_name() for io in other.get_inputs()]) if len(step_outputs) > 0 and not step_outputs.isdisjoint( other_inputs): graph[other.get_name()].add(step.get_name()) edged_nodes.add(step.get_name()) edged_nodes.add(other.get_name()) isolated_nodes = set([step.get_name() for step in steps]) - edged_nodes for node in isolated_nodes: graph[node] = set() ts = TopologicalSorter(graph) return tuple(ts.static_order())
def destCity(self, paths: List[List[str]]) -> str: from graphlib import TopologicalSorter adict = {} for key, value in paths: adict[value] = {key} ts = TopologicalSorter(adict) return tuple(ts.static_order())[-1]
def _sort_steps(steps: Sequence["_Step"]) -> Tuple[str, ...]: """Return a tuple comprised by step names in the order they should be executed.""" graph = defaultdict(set) if len(steps) == 1: return (steps[0].name, ) edged_nodes = set() for step in steps: for other in steps: if step == other: continue step_outputs = set(io.name for io in step.outputs) other_inputs = set(io.name for io in other.inputs) if len(step_outputs) > 0 and not step_outputs.isdisjoint( other_inputs): graph[other.name].add(step.name) edged_nodes.add(step.name) edged_nodes.add(other.name) isolated_nodes = set(step.name for step in steps) - edged_nodes for node in isolated_nodes: graph[node] = set() ts = TopologicalSorter(graph) return tuple(ts.static_order())
def find_models(module: ModuleType) -> List[Type[Model]]: """ Find all models in a migration script. """ models: List[Type[Model]] = [] tables = extract_modified_tables(module) # add models defined explicitly in the migration script queue = list(module.__dict__.values()) while queue: obj = queue.pop() if hasattr(obj, "__tablename__"): tables.add(obj.__tablename__) elif isinstance(obj, list): queue.extend(obj) elif isinstance(obj, dict): queue.extend(obj.values()) # build models by automapping the existing tables, instead of using current # code; this is needed for migrations that modify schemas (eg, add a column), # where the current model is out-of-sync with the existing table after a # downgrade sqlalchemy_uri = current_app.config["SQLALCHEMY_DATABASE_URI"] engine = create_engine(sqlalchemy_uri) Base = automap_base() Base.prepare(engine, reflect=True) seen = set() while tables: table = tables.pop() seen.add(table) model = getattr(Base.classes, table) model.__tablename__ = table models.append(model) # add other models referenced in foreign keys inspector = inspect(model) for column in inspector.columns.values(): for foreign_key in column.foreign_keys: table = foreign_key.column.table.name if table not in seen: tables.add(table) # sort topologically so we can create entities in order and # maintain relationships (eg, create a database before creating # a slice) sorter = TopologicalSorter() for model in models: inspector = inspect(model) dependent_tables: List[str] = [] for column in inspector.columns.values(): for foreign_key in column.foreign_keys: if foreign_key.column.table.name != model.__tablename__: dependent_tables.append(foreign_key.column.table.name) sorter.add(model.__tablename__, *dependent_tables) order = list(sorter.static_order()) models.sort(key=lambda model: order.index(model.__tablename__)) return models
def sort(graph: Graph) -> List[Node]: """ Linearize a dependency graph, throwing an exception if a cycle is detected. When no dependencies intervene in ordering, the algorithm preserves the original insertion order of the graph. :arg graph: a dict mapping each node to an iterable of their antecedents """ sorter = TopologicalSorter(graph) return list(sorter.static_order())
def connections(self) -> List[Callable]: """All connection callbacks registered to this system. Sorted in topological order Returns: List[Callable]: Sorted list of connection callbacks """ return list(TopologicalSorter(self._pre_connections).static_order()) + list( TopologicalSorter(self._post_connections).static_order() )
def get_ready(self): ts = TopologicalSorter() nodes = self.get_all().filter(lambda x: x == "not_running").to_dict( lambda x: x.name) for node in nodes.values(): depends = Linq(node.depends).filter( lambda x: not x == "success").map(lambda x: x.name).to_list() ts.add(node.name, *depends) return Linq(ts.static_order()).map(lambda x: nodes[x]).save()
def _normalize_and_sort_symbols(self): """Removes __init__.py from symbols and returns dict sorted by symbol type""" tmp_sorted = {} topological_sorter = TopologicalSorter() relative_star_imports_volume = {} namespaces = defaultdict(list) for k, v in sorted(self._original_symbols.items(), key=lambda x: x[1]["type"]): new_symbol = k.replace(".__init__", "") if is_relative_import(v): shadows = resolve_relative_import( **{k: v for k, v in v["data"].items() if k in {"module", "level", "shadows"}} ) data = dict(v, data=dict(shadows=shadows)) tmp_sorted[new_symbol] = data namespace, _, symbol = new_symbol.rpartition(".") namespaces[namespace].append(new_symbol) elif is_relative_star_import(v): imports = [ resolve_relative_import(**{k: v for k, v in data.items() if k in {"module", "level", "shadows"}}) for data in v["data"]["imports"] ] data = dict(v, data=dict(imports=imports)) topological_sorter.add(new_symbol, *[f"{imp}.{RELATIVE_IMPORT_IDENTIFIER}.*" for imp in imports]) relative_star_imports_volume[new_symbol] = data namespace, _, symbol = new_symbol.partition(f".{RELATIVE_IMPORT_IDENTIFIER}") namespaces[namespace].append(new_symbol) else: tmp_sorted[new_symbol] = v if get_symbol_type(v) not in {SymbolType.PACKAGE, SymbolType.MODULE}: namespace = new_symbol.rpartition(".")[0] namespaces[namespace].append(new_symbol) try: for rel_import in topological_sorter.static_order(): if rel_import in relative_star_imports_volume: tmp_sorted[rel_import] = relative_star_imports_volume[rel_import] except CycleError: pass shadows_by_in_module_symbol = { k: v["data"]["shadows"] for k, v in tmp_sorted.items() if "shadows" in v["data"] and v["type"] == SymbolType.RELATIVE_IMPORT } for k, v in tmp_sorted.items(): volume = v["data"].get("symbols_in_volume", {}) for volume_symbol in list(volume): no_init_volume_symbol = volume_symbol.replace(".__init__", "") if no_init_volume_symbol in shadows_by_in_module_symbol: new_volume_symbol = shadows_by_in_module_symbol[no_init_volume_symbol] volume[new_volume_symbol] = volume.pop(volume_symbol) self._sorted_symbols = tmp_sorted self._namespaces = namespaces
def _update_extensions(self) -> None: extension_types_enabled_in_configuration = { extension_configuration.extension_type for extension_configuration in self._configuration.extensions if extension_configuration.enabled } extension_types_sorter = TopologicalSorter( _build_extension_type_graph( extension_types_enabled_in_configuration)) extension_types_sorter.prepare() extensions = [] while extension_types_sorter.is_active(): extension_types_batch = extension_types_sorter.get_ready() extensions_batch = [] for extension_type in extension_types_batch: if issubclass(extension_type, ConfigurableExtension): if extension_type not in extension_types_enabled_in_configuration or self._configuration.extensions[ extension_type].extension_type_configuration is None: configuration = extension_type.default_configuration() else: configuration = self._configuration.extensions[ extension_type].extension_type_configuration extension = extension_type(self, configuration) else: extension = extension_type(self) extensions_batch.append(extension) extension_types_sorter.done(extension_type) extensions.append(extensions_batch) self._extensions._update(extensions)
def do_all_tasks(graph, task_queue, done_queue): topological_sorter = TopologicalSorter(graph) topological_sorter.prepare() while topological_sorter.is_active(): for node in topological_sorter.get_ready(): task_queue.put(node) node = done_queue.get() topological_sorter.done(node)
def build(self, step = ''): dep_graph = TopologicalSorter() for x, y in self.packages.items(): dep_graph.add(x, *y.requirements) compile_order = [*dep_graph.static_order()] packages_to_compile_in_order = [] for x in compile_order: packages_to_compile_in_order.append(self.packages[x]) if step == 'reinstall': for pkg in packages_to_compile_in_order: if (not pkg.install(self.config)): print(f"Failed to install {pkg.name}!") return 1 print(f"[nyx]: Installing {pkg.name}-{pkg.version}") return 0 for pkg in packages_to_compile_in_order: if (pkg.cached): print(f"[nyx]: Using cached version of package {pkg.name}-{pkg.version}") continue print(f"[nyx]: Compiling {pkg.name}-{pkg.version}") # Fetch if (not pkg.fetch(self.config)): print(f"Failed to fetch {pkg.name}!") return 1 # Patch if (not pkg.patch(self.config)): print(f"Failed to patch {pkg.name}!") return 1 # Configure if (not pkg.configure(self.config, self.enviroment)): print(f"Failed to configure {pkg.name}!") return 1 # Build if (not pkg.build(self.config, self.enviroment)): print(f"Failed to build {pkg.name}!") return 1 # Package if (not pkg.package(self.config, self.enviroment)): print(f"Failed to package {pkg.name}!") return 1 if (not pkg.install(self.config)): print(f"Failed to install {pkg.name}!") return 1 return 0
def _step(self, time): # Recurse over subsystems for sub in self._subs: sub._step(time) # Apply pre-connections for con in TopologicalSorter(self._pre_connections).static_order(): con(self, time) # Step this system self.do_step(time) # Apply post-connections for con in TopologicalSorter(self._post_connections).static_order(): con(self, time)
def __sort_nodes(self, from_nodes: frozenset['SvNode'] = None, to_nodes: frozenset['SvNode'] = None)\ -> list[tuple['SvNode', list[NodeSocket]]]: """Sort nodes of the tree in proper execution order. Whe all given parameters are None it uses all tree nodes :from_nodes: if given it sorts only next nodes from given ones :to_nodes: if given it sorts only previous nodes from given If from_nodes and to_nodes are given it uses only intersection of next nodes from from_nodes and previous nodes from to_nodes""" nodes_to_walk = set() walk_structure = None if from_nodes is None and to_nodes is None: walk_structure = self._from_nodes elif from_nodes and to_nodes: from_ = self.nodes_from(from_nodes) to_ = self.nodes_to(to_nodes) nodes_to_walk = from_.intersection(to_) elif from_nodes: nodes_to_walk = self.nodes_from(from_nodes) else: nodes_to_walk = self.nodes_to(from_nodes) if nodes_to_walk: walk_structure: dict[SvNode, set[SvNode]] = defaultdict(set) for n in nodes_to_walk: if n in self._from_nodes: walk_structure[n] = {_n for _n in self._from_nodes[n] if _n in nodes_to_walk} nodes = [] if walk_structure: for node in TopologicalSorter(walk_structure).static_order(): nodes.append((node, [self._from_sock.get(s) for s in node.inputs])) return nodes
def validate(self, result: Announcement): graph = TopologicalSorter() for dep in result.dependencies: graph.add(dep) for key, message in self.items(): for dep in message.dependencies: graph.add(dep, key) graph.prepare() missing = graph._node2info.keys() - self.keys() if missing: raise ValueError(f"Missing items: {','.join(missing)}")
def sort_nodes(self, nodes: Iterable['SvNode']) -> list['SvNode']: """Returns nodes in order of their correct execution""" walk_structure: dict[SvNode, set[SvNode]] = defaultdict(set) for n in nodes: if n in self._from_nodes: walk_structure[n] = {_n for _n in self._from_nodes[n] if _n in nodes} nodes = [] for node in TopologicalSorter(walk_structure).static_order(): nodes.append(node) return nodes
def init_graph(self, operations_graph): ts = TopologicalSorter(operations_graph) elements_queue = [*ts.static_order()] for v in elements_queue: self.gather_global_operations(v) for op_name, op in self.global_operations_info.items(): self.global_operations_info[ op_name] = self.replace_all_get_functions(op) for v in elements_queue: (_, element_type, _) = utils.tosca_type_parse(v.type) if element_type == NODES or element_type == RELATIONSHIPS: new_conf_args = self.replace_all_get_functions( v.configuration_args) v.configuration_args = new_conf_args else: del operations_graph[v] for key in operations_graph: if v in operations_graph[key]: operations_graph[key].remove(v) return operations_graph
def find_models(module: ModuleType) -> List[Type[Model]]: """ Find all models in a migration script. """ models: List[Type[Model]] = [] tables = extract_modified_tables(module) # add models defined explicitly in the migration script queue = list(module.__dict__.values()) while queue: obj = queue.pop() if hasattr(obj, "__tablename__"): tables.add(obj.__tablename__) elif isinstance(obj, list): queue.extend(obj) elif isinstance(obj, dict): queue.extend(obj.values()) # add implicit models # pylint: disable=no-member, protected-access for obj in Model._decl_class_registry.values(): if hasattr(obj, "__table__") and obj.__table__.fullname in tables: models.append(obj) # sort topologically so we can create entities in order and # maintain relationships (eg, create a database before creating # a slice) sorter = TopologicalSorter() for model in models: inspector = inspect(model) dependent_tables: List[str] = [] for column in inspector.columns.values(): for foreign_key in column.foreign_keys: dependent_tables.append( foreign_key.target_fullname.split(".")[0]) sorter.add(model.__tablename__, *dependent_tables) order = list(sorter.static_order()) models.sort(key=lambda model: order.index(model.__tablename__)) return models
def topsort(graph): """ Topologically sort a graph The graph should be of the form ``{node: [list of nodes], ...}``. """ try: from graphlib import TopologicalSorter sorter = TopologicalSorter(graph) return list(sorter.static_order()) except ImportError: # TODO: Can be removed when we depend on Python >= 3.9 # make a copy so as not to destroy original graph = dict((k, copy(v)) for k, v in graph.items()) # Use the standard algorithm for topological sorting: # http://en.wikipedia.org/wiki/Topological_sorting # List that will contain the sorted elements sorted_items = [] # set of all nodes with no incoming edges: no_incoming = { node for node, edges in graph.items() if len(edges) == 0 } while len(no_incoming): n = no_incoming.pop() sorted_items.append(n) # find nodes m with edges to n outgoing = [m for m, edges in graph.items() if n in edges] for m in outgoing: graph[m].remove(n) if len(graph[m]) == 0: # no other dependencies no_incoming.add(m) if any([len(edges) > 0 for edges in graph.values()]): raise ValueError('Cannot topologically sort cyclic graph.') return sorted_items
def _clean_places(ancestry: Ancestry) -> None: places = ancestry.entities[Place] def _extend_place_graph(graph: Dict, enclosing_place: Place) -> None: enclosures = enclosing_place.encloses # Ensure each place appears in the graph, even if they're anonymous. graph.setdefault(enclosing_place, set()) for enclosure in enclosures: enclosed_place = enclosure.encloses seen_enclosed_place = enclosed_place in graph graph[enclosing_place].add(enclosed_place) if not seen_enclosed_place: _extend_place_graph(graph, enclosed_place) places_graph = defaultdict(set) for place in places: _extend_place_graph(places_graph, place) for place in TopologicalSorter(places_graph).static_order(): _clean_place(ancestry, place)
def count_containing_bags(rules: Rules, central_color: str) -> int: """ Counts the number of bags which are contained within a bag of the given central color. """ adjlist = extract_adjlist(rules) contained_bags = reachable_nodes(adjlist, central_color) # TopologicalSorter requires the graph in the reversed direction reversed_adjlist = { k: v for k, v in extract_reversed_adjlist(rules).items() if k in contained_bags } total_counts = collections.Counter() total_counts[central_color] = 1 for curr in TopologicalSorter(reversed_adjlist).static_order(): for color, count in rules[curr].items(): total_counts[color] += count * total_counts[curr] return sum(total_counts.values()) - 1
import fileinput as fi import heapq from graphlib import TopologicalSorter ts = TopologicalSorter() for line in fi.input(): pr = line.split(" ") a, b = pr[1], pr[-3] ts.add(b, a) ts.prepare() EXTRA = 60 # Worker heaps IDLE = 5 BUSY = [] Q = list(ts.get_ready()) heapq.heapify(Q) t_min = 0 while ts.is_active(): # While there are tasks and idle workers, we hand out tasks. while Q and IDLE > 0: c = heapq.heappop(Q) IDLE -= 1 heapq.heappush(BUSY, (t_min + ord(c) - (ord("A") - 1) + EXTRA, c))
logger.error(f'Failed to load %s', pkgbase) traceback.print_exc() failed = [i.key for i in Status.objects.filter(status='ERROR')] for i in failed: recursively_fail(dependency_graph, reversed_dependency_graph, i) staled = [i.key for i in Status.objects.filter(status='STALED')] for i in staled: recursively_skip(dependency_graph, reversed_dependency_graph, i) building = [i.key for i in Status.objects.filter(status='BUILDING')] for i in building: recursively_skip(dependency_graph, reversed_dependency_graph, i) sorter = TopologicalSorter(dependency_graph) order = list(sorter.static_order()) order = [i for i in order if i in staled] resources = config['scheduler'] for group in resources.keys(): if group == 'default': continue resources[group]['used'] = Status.objects.filter( detail__startswith=group).count() logger.info('%s: %d / %d', group, resources[group]['used'], resources[group]['total']) for i in order: if i == 'dummy':
'myprog': ['extlib1', 'fmtlib', 'urllib', 'mylib'], 'mylib': ['fmtlib', 'stdlib'], } do_all_tasks(graph, task_queue, done_queue) for i in range(NUMBER_OF_PROCESSES): task_queue.put('STOP') def do_all_tasks(graph, task_queue, done_queue): topological_sorter = TopologicalSorter(graph) topological_sorter.prepare() while topological_sorter.is_active(): for node in topological_sorter.get_ready(): task_queue.put(node) node = done_queue.get() topological_sorter.done(node) if __name__ == '__main__': dependencies = { 'scipy': {'numpy'}, 'pandas': {'numpy', 'scipy', 'requests'}, 'requests': {'urllib3'}, } from graphlib import TopologicalSorter ts = TopologicalSorter(dependencies) print(list(ts.static_order()))
from graphlib import TopologicalSorter # LIBRARY mapped_to LIBRARY DEPENDENCIES data = { 'des_system_lib': set('std synopsys std_cell_lib des_system_lib dw02 dw01 ramlib ieee'.split()), 'dw01': set('ieee dw01 dware gtech'.split()), 'dw02': set('ieee dw02 dware'.split()), 'dw03': set('std synopsys dware dw03 dw02 dw01 ieee gtech'.split()), 'dw04': set('dw04 ieee dw01 dware gtech'.split()), 'dw05': set('dw05 ieee dware'.split()), 'dw06': set('dw06 ieee dware'.split()), 'dw07': set('ieee dware'.split()), 'dware': set('ieee dware'.split()), 'gtech': set('ieee gtech'.split()), 'ramlib': set('std ieee'.split()), 'std_cell_lib': set('ieee std_cell_lib'.split()), 'synopsys': set(), } # Ignore self dependencies for k, v in data.items(): v.discard(k) ts = TopologicalSorter(data) print(tuple(ts.static_order()))
from graphlib import TopologicalSorter """ |- SW11 |- SW2 -|- SW12 SW1 -| |- SW3 -|- SW13 |- SW14 """ topology = { "SW1": ["SW2", "SW3"], "SW2": ["SW11", "SW12"], "SW3": ["SW13", "SW14"] } top = TopologicalSorter(topology) print(list(top.static_order())) # ['SW11', 'SW12', 'SW13', 'SW14', 'SW2', 'SW3', 'SW1']
"Подготовка VM": [ "Обновить Python до 3.9", "Клонировать репозиторий курса", "Установить pyneng.py", ], "Клонировать репозиторий курса": [ "Подготовка репозитория", ], "Подготовка репозитория": [ "Исправить ошибки в тестах", "Исправить ошибки в заданиях", ], "Исправить ошибки в тестах": [ "Изменить в тестах порядок в assert", "Вынести общие функции в pyneng_common_functions", ], "Исправить ошибки в заданиях": ["Исправить ошибки в заданиях из списка todo"], } tasks = TopologicalSorter(tasks_dict) pprint(list(tasks.static_order())) [ 'Обновить Python до 3.9', 'Установить pyneng.py', 'Изменить в тестах порядок в assert', 'Вынести общие функции в pyneng_common_functions', 'Исправить ошибки в заданиях из списка todo', 'Исправить ошибки в тестах', 'Исправить ошибки в заданиях', 'Подготовка репозитория', 'Клонировать репозиторий курса', 'Подготовка VM' ]
def to_dsl(self, provider, operations_graph, reversed_operations_graph, cluster_name, is_delete, artifacts=None, target_directory=None, inputs=None, outputs=None, extra=None, debug=False): if artifacts is None: artifacts = [] if target_directory is None: target_directory = self.initial_artifacts_directory self.artifacts = {} for art in artifacts: self.artifacts[art[NAME]] = art provider_config = ProviderConfiguration(provider) ansible_config = provider_config.get_section(ANSIBLE) node_filter_config = provider_config.get_subsection( ANSIBLE, NODE_FILTER) ids_file_path = os.getcwd( ) + '/id_vars_' + cluster_name + self.get_artifact_extension() self.init_global_variables(inputs) operations_graph = self.init_graph(operations_graph) # the graph of operations at the moment is a dictionary of copies of ProviderTemplatre objects, # of the form Node/Relationship: {the set of opers of Nodes/Relationships on which it depends} elements = TopologicalSorter(operations_graph) # use TopologicalSorter for creating graph if is_delete: reversed_operations_graph = self.init_graph( reversed_operations_graph) elements = TopologicalSorter(reversed_operations_graph) elements.prepare() # first operations from on top of the graph in state 'ready' ansible_playbook = [] if not debug: self.prepare_for_run() # function for initializing tmp clouni directory q = Queue() # queue for node names + operations active = [] # list of parallel active operations first = True while elements.is_active(): node_name = None # try to get new finished operation from queue and find it in list of active # if get - mark done this operation (but before it find in graph) # if ready operations exists - get it and execute, remove from active try: node_name = q.get_nowait() except: time.sleep(1) if node_name is not None: for node in active: if node.name == node_name.split( SEPARATOR )[0] and node.operation == node_name.split(SEPARATOR)[1]: active.remove(node) elements.done(node) for v in elements.get_ready(): # in delete mode we skip all operations exept delete and create operation transforms to delete if is_delete: if v.operation == 'create': v.operation = 'delete' else: elements.done(v) continue logging.debug("Creating ansible play from operation: %s" % v.name + ':' + v.operation) extra_tasks_for_delete = self.get_extra_tasks_for_delete( v.type, v.name.replace('-', '_'), ids_file_path) description_prefix, module_prefix = self.get_module_prefixes( is_delete, ansible_config) description_by_type = self.ansible_description_by_type( v.type_name, description_prefix) module_by_type = self.ansible_module_by_type( v.type_name, module_prefix) ansible_play_for_elem = dict(name=description_prefix + ' ' + provider + ' cluster: ' + v.name + ':' + v.operation, hosts=self.default_host, tasks=[]) # reload id_vars file if not is_delete and first: first = False ansible_play_for_elem['tasks'].append( copy.deepcopy( {FILE: { PATH: ids_file_path, STATE: 'absent' }})) ansible_play_for_elem['tasks'].append( copy.deepcopy( {FILE: { PATH: ids_file_path, STATE: 'touch' }})) # create playbook for every operation if v.operation == 'delete': if not v.is_software_component: ansible_play_for_elem['tasks'].append( copy.deepcopy({'include_vars': ids_file_path})) ansible_tasks = self.get_ansible_tasks_for_delete( v, description_by_type, module_by_type, additional_args=extra) ansible_tasks.extend( self.get_ansible_tasks_from_interface( v, target_directory, is_delete, v.operation, cluster_name, additional_args=extra)) if not any(item == module_by_type for item in ansible_config.get( 'modules_skipping_delete', [])): ansible_play_for_elem['tasks'].extend( copy.deepcopy(ansible_tasks)) elif v.operation == 'create': if not v.is_software_component: ansible_play_for_elem['tasks'].extend( copy.deepcopy( self.get_ansible_tasks_for_inputs(inputs))) ansible_tasks = self.get_ansible_tasks_for_create( v, target_directory, node_filter_config, description_by_type, module_by_type, additional_args=extra) ansible_tasks.extend( self.get_ansible_tasks_from_interface( v, target_directory, is_delete, v.operation, cluster_name, additional_args=extra)) ansible_play_for_elem['tasks'].extend( copy.deepcopy(ansible_tasks)) ansible_play_for_elem['tasks'].extend( copy.deepcopy(extra_tasks_for_delete)) else: ansible_play_for_elem['hosts'] = v.host ansible_play_for_elem['tasks'].extend( copy.deepcopy( self.get_ansible_tasks_from_interface( v, target_directory, is_delete, v.operation, cluster_name, additional_args=extra))) else: (_, element_type, _) = utils.tosca_type_parse(v.type) if element_type == NODES: if v.is_software_component: ansible_play_for_elem['hosts'] = v.host # operations for relationships executes on target/source host depends on operation elif element_type == RELATIONSHIPS: if v.operation == 'pre_configure_target' or v.operation == 'post_configure_target' or v.operation == 'add_source': for elem in operations_graph: if elem.name == v.target: if elem.is_software_component: ansible_play_for_elem[ 'hosts'] = elem.host break elif v.operation == 'pre_configure_source' or v.operation == 'post_configure_source': for elem in operations_graph: if elem.name == v.source: if elem.is_software_component: ansible_play_for_elem[ 'hosts'] = elem.host break else: logging.error( "Unsupported operation for relationship in operation graph" ) sys.exit(1) else: logging.error( "Unsupported element type in operation graph") sys.exit(1) ansible_play_for_elem['tasks'].extend( copy.deepcopy( self.get_ansible_tasks_from_interface( v, target_directory, is_delete, v.operation, cluster_name, additional_args=extra))) ansible_playbook.append(ansible_play_for_elem) # run playbooks if not debug: self.parallel_run([ansible_play_for_elem], v.name, v.operation, q, cluster_name) # add element to active list active.append(v) else: elements.done(v) if is_delete: last_play = dict(name='Renew id_vars_example.yaml', hosts=self.default_host, tasks=[]) last_play['tasks'].append( copy.deepcopy({FILE: { PATH: ids_file_path, STATE: 'absent' }})) if not debug: self.parallel_run([last_play], None, None, q, cluster_name) done = q.get() if done != 'Done': logging.error("Something wrong with multiprocessing queue") sys.exit(1) ansible_playbook.append(last_play) # delete dir with cluster_name in tmp clouni dir if not debug: rmtree(os.path.join(utils.get_tmp_clouni_dir(), cluster_name)) return yaml.dump(ansible_playbook, default_flow_style=False, sort_keys=False)
from graphlib import TopologicalSorter table_references = { "customers": set(), "accounts": {"customers"}, "products": set(), "orders": {"accounts", "customers"}, "order_products": {"orders", "products"}, } if __name__ == "__main__": sorter = TopologicalSorter(table_references) print(list(sorter.static_order()))
def iter_results(toposorter: graphlib.TopologicalSorter): while toposorter.is_active(): ready_tasks = tuple(toposorter.get_ready()) toposorter.done(*ready_tasks) yield ready_tasks
def get_op_order(self): ts = TopologicalSorter() for host in self.inventory: for i, op_hash in enumerate(host.op_hash_order): if not i: ts.add(op_hash) else: ts.add(op_hash, host.op_hash_order[i - 1]) final_op_order = [] ts.prepare() while ts.is_active(): # Ensure that where we have multiple different operations that can be executed in any # dependency order we order them by line numbers. node_group = sorted( ts.get_ready(), key=lambda op_hash: self.op_meta[op_hash]["op_order"], ) ts.done(*node_group) final_op_order.extend(node_group) return final_op_order