def test_simple(self): self.assertEqual( list( toposort({ 2: {11}, 9: {11, 8}, 10: {11, 3}, 11: {7, 5}, 8: {7, 3}, })), [ {3, 5, 7}, {8, 11}, {2, 9, 10}, ]) # make sure self dependencies are ignored self.assertEqual( list( toposort({ 2: {2, 11}, 9: {11, 8}, 10: {10, 11, 3}, 11: {7, 5}, 8: {7, 3}, })), [ {3, 5, 7}, {8, 11}, {2, 9, 10}, ]) self.assertEqual(list(toposort({1: set()})), [{1}]) self.assertEqual(list(toposort({1: {1}})), [{1}])
def ordered_steps(self): dependencies = { step.name: step.depends() for step in self.steps() } # add dependencies on trait-defined steps for step in self.steps(): dependencies[step.name] |= { s.name for s in self.steps() if s.injecting_trait_name() in step.trait_depends() } try: result = list(toposort.toposort(dependencies)) except toposort.CircularDependencyError as de: dependencies = self._find_and_resolve_publish_trait_circular_dependencies( dependencies, cycle_info=de.data, ) dependencies = self._find_and_resolve_release_trait_circular_dependencies( dependencies, cycle_info=de.data, ) # try again - if there is still a cyclic dependency, this is probably caused # by a user error - so let it propagate result = list(toposort.toposort(dependencies)) # result contains a generator yielding tuples of step name in the correct execution order. # each tuple can/should be parallelised return result
def test_simple(self): self.assertEqual(list(toposort({2: {11}, 9: {11, 8}, 10: {11, 3}, 11: {7, 5}, 8: {7, 3}, })), [{3, 5, 7}, {8, 11}, {2, 9, 10}, ]) # make sure self dependencies are ignored self.assertEqual(list(toposort({2: {2, 11}, 9: {11, 8}, 10: {10, 11, 3}, 11: {7, 5}, 8: {7, 3}, })), [{3, 5, 7}, {8, 11}, {2, 9, 10}, ]) self.assertEqual(list(toposort({1: set()})), [{1}]) self.assertEqual(list(toposort({1: {1}})), [{1}])
def test_cycle(self): # a simple, 2 element cycle self.assertRaises(ValueError, list, toposort({1: {2}, 2: {1} })) # an indirect cycle self.assertRaises(ValueError, list, toposort({1: {2}, 2: {3}, 3: {1}, }))
def eval_order_and_gate_lookup(self): deps = fn.merge( {a & -2: {b & -2, c & -2} for a, b, c in self.gates}, {a & -2: set() for a in self.inputs.values()}, {a & -2: set() for a, _, _ in self.latches.values()}, # LatchIn ) latch_deps = {a & -2: {b & -2} for a, b, _ in self.latches.values()} return list(toposort(deps)), list(toposort(latch_deps))
def test_no_dependencies(self): self.assertEqual(list(toposort({1: {2}, 3: {4}, 5: {6}, })), [{2, 4, 6}, {1, 3, 5}]) self.assertEqual(list(toposort({1: set(), 3: set(), 5: set(), })), [{1, 3, 5}])
def test_no_dependencies(self): self.assertEqual(list(toposort({ 1: {2}, 3: {4}, 5: {6}, })), [{2, 4, 6}, {1, 3, 5}]) self.assertEqual(list(toposort({ 1: set(), 3: set(), 5: set(), })), [{1, 3, 5}])
def get_all_classes_that_needs_schema(class_: type) -> List[type]: """ Iterates over the "class_" attributes searching for objects that needs schema (attr classes) :param type class_: Target class to return all elements that needs schema. """ graph = {} search_queue = deque() graph[CaseDescription] = _get_classes(class_) search_queue += graph[CaseDescription] while search_queue: attr_class = search_queue.popleft() attr_classes = _get_classes(attr_class) graph[attr_class] = attr_classes search_queue += graph[attr_class] # Topological Sorting to keep dependencies correct from toposort import toposort result = [] for dependencies in toposort(graph): # Items are sorted to make the results deterministic result.extend(sorted(dependencies, key=operator.attrgetter("__name__"))) return result
def find_min_topology(self): #, ops, topology=[]): dep_satisfied = lambda dep: ( dep._cacheable and self.using_cache and self.cache is not None and (dep in self.cache or dep.get_name() in self.cache)) dep_map = dict() deps_to_resolve = list(self.required_outputs) while len(deps_to_resolve) > 0: o = deps_to_resolve[0] deps_to_resolve = deps_to_resolve[1:] reqs = self.dep_map[o] # No matter what, the dag needs to see the requirements dep_map[o] = reqs # reqs can be kwargs or args (list) _iter_reqs = reqs.values() if isinstance(reqs, dict) else reqs # But only need to resolve those that are not satisfied already deps_to_resolve += [r for r in _iter_reqs if not dep_satisfied(r)] dep_sets = { p: set(d.values()) if isinstance(d, dict) else set(d) for p, d in dep_map.items() } topology = list(toposort(dep_sets)) return topology
def test_dependencies_list(self): filename = "test_data.json" target_function = "F" try: # Open file with list of dependencies written as a dictionary and multiple correct solutions with open(filename, "r") as data: list_of_results = [] dictionary = ast.literal_eval(data.read()) # Using a global variable TestDependenciesList.filesize = len(dictionary) # Getting just the first element with the list of dependencies list_of_dicts = list(toposort(dictionary[0])) # Create async loop loop = asyncio.get_event_loop() list_of_results = loop.run_until_complete(print_dependencies(list_of_dicts, target_function)) # Ignoring first element of the file (Not part of the possible solutions) for dict in dictionary[1:]: # Without a subtest, execution would stop after the first failure with self.subTest(dict=dict): self.assertEqual(list_of_results, dict) return (dictionary[1:]) except FileNotFoundError: print("File not found") else: data.close()
def arrangeNodes(dg): ts = list(toposort.toposort(dg)) xc = 0 ips = {} for st in ts: yps = []; st = list(st) for si in st: if si in dg.keys(): ri = list(dg[si]) yp = np.mean([ips[rr][1] for rr in ri]) else: yp = 0 while yp in yps: yp += 1 yps.append(yp) #ysi = np.argsort(yps) #ys = (np.arange(len(ysi)) - ysi.mean()) ysi = np.arange(len(yps)) ys = yps for i, yi in zip(ysi, ys): ips[st[i]] = (xc, yi) xc += 1 return ips
def load_schema(self): from metaphor.lrparse.lrparse import parse self.specs = {} self.root = Spec('root', self) schema_data = self._load_schema_data() self._build_specs(schema_data) calcs = self._collect_calcs(schema_data) calc_deps = dict( (key, set(calc.get('deps', {}))) for key, calc in calcs.items()) sorted_calcs = list(toposort(calc_deps)) for line in sorted_calcs: for field_str in line: spec_name, field_name = field_str.split('.') field_data = schema_data['specs'][spec_name]['fields'][ field_name] if field_data['type'] == 'calc': spec = self.specs[spec_name] # log.debug("Calc create: %s.%s", spec_name, field_name) self._add_calc(spec, field_name, field_data['calc_str']) self.calc_trees[(spec.name, field_name)] = parse( field_data['calc_str'], spec) for root_name, root_data in schema_data.get('root', {}).items(): self._add_field(self.root, root_name, root_data['type'], target_spec_name=root_data.get('target_spec_name'))
def test_sort_flatten(self): data = { 2: {11}, 9: {11, 8}, 10: {11, 3}, 11: {7, 5}, 8: {7, 3, 8}, # includes something self-referential } expected = [{3, 5, 7}, {8, 11}, {2, 9, 10}] self.assertEqual(list(toposort(data)), expected) # now check the sorted results results = [] for item in expected: results.extend(sorted(item)) self.assertEqual(toposort_flatten(data), results) # and the unsorted results. break the results up into groups to compare them actual = toposort_flatten(data, False) results = [{i for i in actual[0:3]}, {i for i in actual[3:5]}, {i for i in actual[5:8]}] self.assertEqual(results, expected)
def update_queue(package_queue, to_build_orig, failed, in_progress, built): if failed: # If we have failure(s), there is no point in continuing return to_build = {k: v for k, v in to_build_orig.items()} to_remove = set() for pkg_name, package in in_progress.items(): for child in package.children: to_remove.add(child) for rm in to_remove: to_build.pop(rm, None) deps_mapping = { p.name: {d for d in p.build_time_dependencies() if d not in built} for p in list(to_build.values()) } sorted_ordering = [list(deps) for deps in toposort(deps_mapping)] for item in filter( lambda i: i in to_build and i not in in_progress and i not in package_queue.queue and i not in built, sorted(sorted_ordering[0], key=lambda k: to_build_orig[k].batch_priority) if sorted_ordering else []): package_queue.put(to_build_orig.pop(item))
def topo_sort(nodes): n = [node.id for node in nodes] n_dict = dict() if None in n: logger.error("node with id None") for i, node in enumerate(nodes): dep = [] for arg in node.inputs: for val in arg: if val["id"] in n: dep.append(n.index(val["id"])) n_dict[i] = set(dep) sorted_nodes = list(toposort(n_dict)) re = [] for rank in sorted_nodes: for i in list(rank): re.append(nodes[i]) return re
def test_toposort(): tf.reset_default_graph() nodes = util.make_caterpillar_graph(length=2) graph = linearize_lib.get_graph() initial = list(toposort(graph))[0] assert len(initial) == 1 assert list(initial)[0].name == 'merge2'
def __init__(self, node_type, adj_mat, n_channels): """ With a valid and reduced network topology (adj_mat), a cell is constructed :param node_type: 1d np.array node_type[2xi] in (conv, pool[id]), node_type[2xi + 1] in (1by1, 3by3) :param adj_mat: 2d np.array upper triangular with zero diagonal (i,j) entry represents edge from i-node to j-node :param n_channels: numeric(int) """ super(NASBinaryCell, self).__init__() self.n_nodes = adj_mat.shape[0] assert len(node_type) == (self.n_nodes - 2) * 2 assert (0 <= node_type).all() and (node_type <= 1).all() self.adj_mat = adj_mat topo_input = {v: set(np.where(adj_mat[:, v] == 1)[0]) for v in range(1, self.n_nodes) if np.sum(adj_mat[:, v]) > 0} self.topo_order = list(toposort.toposort(topo_input)) assert self.topo_order[0] == {0} assert self.topo_order[-1] == {self.n_nodes - 1} self.node0 = None for i in range(1, self.n_nodes - 1): n_t, n_s = node_type[2*i-2:2*i] node = (NodeId() if n_s == 0 else NodeMaxpool3by3()) if n_t == 0 else \ (NodeConv3by3(n_channels) if n_s == 0 else NodeConv5by5(n_channels)) setattr(self, 'node' + str(i), node) setattr(self, 'node' + str(self.n_nodes - 1), NodeId())
def split_dags(forest): parents = defaultdict(set) def find_parents(expr): for child in expr.children: parents[child].add(expr) find_parents(child) for expr in forest: find_parents(expr) def contains(tree, expr): if tree == expr: return True else: return any(contains(child, expr) for child in tree.children) exprs = forest + [expr for expr in parents if len(parents[expr]) > 1] # TODO: This is O(n^3) deps = {expr: set() for expr in exprs} for i in xrange(len(exprs)): for j in xrange(len(exprs)): if i == j: continue if contains(exprs[i], exprs[j]): deps[exprs[i]].add(exprs[j]) return list(toposort(deps))
def _topologically_sorted(node_dependencies) -> List[Set[Node]]: """Topologically group and sort (order) nodes such that no node depends on a node that appears in the same or a later group. Raises: CircularDependencyError: When it is not possible to topologically order provided nodes. Returns: The list of node sets in order of execution. First set is nodes that should be executed first (no dependencies), second set are nodes that should be executed on the second step, etc. """ def _circle_error_message(error_data: Dict[str, str]) -> str: """Error messages provided by the toposort library will refer to indices that are used as an intermediate step. This method can be used to replace that message with one that refers to the nodes' string representations. """ circular = [str(node) for node in error_data.keys()] return "Circular dependencies exist among these items: {}".format(circular) try: return list(toposort(node_dependencies)) except ToposortCircleError as error: message = _circle_error_message(error.data) raise CircularDependencyError(message) from error
def _construct_toposorted_chain_of_tasks(self) -> Chain: """Take the directed graph and toposort so that we can efficiently organize our workflow, i.e. parallelize where possible. if we have 2 elements where one of both is an Ellipsis object we need to orchestrate just 1 job. In the other case we will loop over the toposorted dag and assign a stepfunctions task or assign multiple tasks in parallel. Returns: toposorted chain of tasks """ self.chain_of_tasks = Chain() directed_graph_toposorted = list(toposort.toposort( self.directed_graph)) if self._is_one_task( directed_graph_toposorted=directed_graph_toposorted): sfn_task = self.add_task(next(iter(directed_graph_toposorted[0]))) self.chain_of_tasks.append(sfn_task) else: for element in directed_graph_toposorted: if len(element) == 1: sfn_task = self.add_task(next(iter(element))) elif len(element) > 1: sfn_task = self.add_parallel_tasks(element) else: raise StepfunctionsWorkflowException( "cannot have an index in the directed graph with 0 elements" ) self.chain_of_tasks.append(sfn_task) return self.chain_of_tasks
def build(self): """Build a topological order """ topo_sort = list(toposort.toposort(self._build_dependency_dict())) for i in range(0, len(topo_sort)): self._topo_sort[i] = topo_sort[i] # if a bw op has the same order with a fw op, # then remove the bw op self._clean_bw_ops() # if there are non-bw ops in the bw phase, # then remove them, and do reordering self._clean_update_ops() self._reindex() # build a dict of (op, order) self._build_order_dict() # starting order of the backward phase for i in range(0, len(self._topo_sort)): ops = self._topo_sort[i] if (ops & self._grad_ops): self._bw_starting_order = i break
def build_dep_graph(input_files, build_info): pattern = re.compile('[a-zA-Z0-9_-]+') flatten = lambda l: [item for sublist in l for item in sublist] dist_to_lines = collections.defaultdict(list) for i in input_files: with open(i) as f: for l in f.readlines(): l = l.strip() m = pattern.match(l) if m: dist_to_lines[m.group()].append(l) if not build_info: return [flatten(dist_to_lines.values())] deps = collections.defaultdict(list) for dist, info in build_info.items(): for d in info.get("additional_buildtime_deps", []): deps[dist] += dist_to_lines[d] graph = { r: set(deps[n]) if n in deps else set() for n, rr in dist_to_lines.items() for r in rr } result = list(toposort.toposort(graph)) return result
def critical_paths(graph): def events_earliest_time(vnum, graph, toposeq): ee = [0] * vnum for i in toposeq: for j, w in graph.out_edges(i): if ee[i] + w > ee[j]: ee[j] = ee[i] + w return ee def event_latest_time(vnum, graph, toposeq, eelast): le = [eelast] * vnum for k in range(vnum-2, -1, -1): i = toposeq[k] for j, w in graph.out_edges(i): if le[j] - w < le[i]: le[i] = le[j] - w return le def crt_paths(vnum, graph, ee, le): crt_actions = [] for i in range(vnum): for j, w in graph.out_edges(i): if ee[i] == le[j] - w: crt_actions.append((i, j, ee[i])) return crt_actions toposeq = toposort(graph) if not toposeq: # 不存在拓扑序列,失败 return False vnum = graph.vertex_num() ee = events_earliest_time(vnum, graph, toposeq) le = event_latest_time(vnum, graph, toposeq, ee[vnum-1]) return crt_paths(vnum, graph, ee, le)
def build_dep_graph(args): pattern = re.compile('[a-zA-Z0-9_-]+') requirements = {} for i in args.input: with open(i) as f: for l in f.readlines(): l = l.strip() m = pattern.match(l) if m: requirements[m.group()] = l if not args.build_dep: return [requirements.values()] deps = {} for i in args.build_dep: k, v = i.split('=') if k not in requirements: continue if k not in deps: deps[k] = [] deps[k].append(requirements[v]) graph = { r: set(deps[n]) if n in deps else set() for n, r in requirements.items() } result = list(toposort.toposort(graph)) return result
def run(self): if not self.run_order: self.run_order = list(chain.from_iterable(toposort(self.adjList))) for n in self.run_order: # Each node to be processed n.next_frame() n.run()
def _sort_tasks(task_list): """ Take the list of tasks in any order and sort them topologically so the resulting list and be executed correctly. :param task_list: List of LocalTask objects :return: Sorted list of LocalTask objects """ # Create a list of tasks_w_deps = {} for task in task_list: deps = [] for port_name in task.inputs.portnames: port = task.inputs.__getattribute__(port_name) if isinstance(port.value, local_task.Directory): if port.value.parent not in task_list: raise LocalWorkflowError( 'Task %s is missing from workflow' % port.value.parent.type) deps.append(port.value.parent) tasks_w_deps[task] = set(deps) sorted_list = [] for x in toposort(tasks_w_deps): sorted_list += list(x) return sorted_list
def __str__(self): if not len(self._memories_dictionary): return "<empty graph>" # Sort memories topologically def get_dependencies(n): return set(n.precursor_keys) to_be_topologically_sorted = {memory.key: get_dependencies(memory) for memory in self._memories_dictionary.values()} topologically_sorted = [l for g in toposort(to_be_topologically_sorted) for l in g] # Find longest strings so we can pad our strings to equal length later def get_precursors_str(n): return ', '.join(self._memories_dictionary[n].precursor_keys) data_to_print = [(get_precursors_str(n), n, self._memories_dictionary[n].is_stale) for n in topologically_sorted] longest_precursors_str = max([len(d[0]) for d in data_to_print]) longest_memory_key = max([len(d[1]) for d in data_to_print]) # Build output str in format: "[precursors] --> memory" along with an indicate of stale-ness result_str = "" for data in data_to_print: left_side = data[0].rjust(longest_precursors_str) right_side = data[1].ljust(longest_memory_key) stale_indicator = " (stale)" if data[2] else "" result_str += f"{left_side} --> {right_side}{stale_indicator}\n" result_str = result_str.rstrip('\n') return result_str
def build_dep_graph(args): pattern = re.compile('[a-zA-Z0-9_-]+') flatten = lambda l: [item for sublist in l for item in sublist] dist_to_lines = collections.defaultdict(list) for i in args.input: with open(i) as f: for l in f.readlines(): l = l.strip() m = pattern.match(l) if m: dist_to_lines[m.group()].append(l) if not args.build_dep: return [flatten(dist_to_lines.values())] deps = collections.defaultdict(list) for i in args.build_dep: k, v = i.split('=') if k not in dist_to_lines: continue deps[k] += dist_to_lines[v] graph = { r: set(deps[n]) if n in deps else set() for n, rr in dist_to_lines.items() for r in rr } result = list(toposort.toposort(graph)) return result
def remove_correlated_features(df, r=0.99): names_correlated = find_correlated_features(df, r=r) sorted_names = list(toposort(names_correlated)) removed_features = list() for names in sorted_names: for name in names: # If feature not on the left - remove it if name not in names_correlated.keys(): for key, value in names_correlated.items(): try: names_correlated[key].remove(name) removed_features.append(name) except KeyError: pass for name in names_correlated.keys(): if not names_correlated[name]: for key, value in names_correlated.items(): try: names_correlated[key].remove(name) removed_features.append(name) names_correlated.pop(name) except KeyError: pass names = list(set(removed_features)) print("Removed features : ") print(names) return df.drop(names, axis=1)
def toposort_symbols(symbols: SymbolDef, field: Optional[str] = None) -> SymbolDef: """ Topologically sort symbol definitions according to their interdependency :param symbols: symbol definitions :param field: field of definition.values() that is used to compute interdependency :return: ordered symbol definitions """ sorted_symbols = toposort({ identifier: { s for s in ( definition[field] if field is not None else definition ).free_symbols if s in symbols } for identifier, definition in symbols.items() }) return { s: symbols[s] for symbol_group in sorted_symbols for s in symbol_group }
def arrangeNodes(dg): ts = list(toposort.toposort(dg)) xc = 0 ips = {} for st in ts: yps = [] st = list(st) for si in st: if si in dg.keys(): ri = list(dg[si]) yp = np.mean([ips[rr][1] for rr in ri]) else: yp = 0 while yp in yps: yp += 1 yps.append(yp) #ysi = np.argsort(yps) #ys = (np.arange(len(ysi)) - ysi.mean()) ysi = np.arange(len(yps)) ys = yps for i, yi in zip(ysi, ys): ips[st[i]] = (xc, yi) xc += 1 return ips
def _sort_tasks(task_list): """ Take the list of tasks in any order and sort them topologically so the resulting list and be executed correctly. :param task_list: List of LocalTask objects :return: Sorted list of LocalTask objects """ # Create a list of tasks_w_deps = {} for task in task_list: deps = [] for port_name in task.inputs.portnames: port = task.inputs.__getattribute__(port_name) if isinstance(port.value, local_task.Directory): if port.value.parent not in task_list: raise LocalWorkflowError('Task %s is missing from workflow' % port.value.parent.type) deps.append(port.value.parent) tasks_w_deps[task] = set(deps) sorted_list =[] for x in toposort(tasks_w_deps): sorted_list += list(x) return sorted_list
def get_topology(self) -> List[Set[EntityType]]: map_of_dependencies = {} for entity_type in self.included_entity_types: map_of_dependencies[entity_type] = [] for target in self.get_dependencies(entity_type, False): map_of_dependencies[entity_type].append(target) return list(toposort(map_of_dependencies))
def main(): args = parse_args() logging.basicConfig(level=args.loglevel) if has_path is False or has_toposort is False: LOG.error('Your python environment is missing required modules') sys.exit(1) dirs = {} for item in (Path(dir) for dir in args.dirs): if item.name.startswith('.'): continue if not item.is_dir(): LOG.warn('skipping %s (not a directory)', item) continue dirs[item] = { 'requires': [], 'required_by': [], 'provides': [], } this = dirs[item] for reqtype in this.keys(): LOG.debug('check %s %s', item.name, reqtype) try: with (item / '.deps' / reqtype).open() as fd: LOG.debug('reading %s deps for %s', reqtype, item.name) this[reqtype] = fd.read().splitlines() except IOError: continue # generated provided_by dictionary provided_by = {} for k, v in dirs.items(): provided_by[k.name] = k for provide in v['provides']: provided_by[provide] = k order = {} for k, v in dirs.items(): order.setdefault(k, set()) for req in v['requires']: LOG.debug('%s requires %s (provided by %s)', k, req, provided_by[req]) order[k].add(provided_by[req]) for req in v['required_by']: LOG.debug('%s is required by %s (via %s)', k, provided_by[req], req) order.setdefault(provided_by[req], set()).add(k) order = [item for stage in toposort.toposort(order) for item in stage] print ' '.join(str(x) for x in order)
def get_relation_view(): _views = PreferenceRelationView.get_by(to_dict=True) views = [] if current_app.config.get("USE_ACL"): for i in _views: try: if ACLManager().has_permission(i.get('name'), ResourceTypeEnum.RELATION_VIEW, PermEnum.READ): views.append(i) except AbortException: pass else: views = _views view2cr_ids = dict() result = dict() name2id = list() for view in views: view2cr_ids.setdefault(view['name'], []).extend(json.loads(view['cr_ids'])) name2id.append([view['name'], view['id']]) id2type = dict() for view_name in view2cr_ids: for i in view2cr_ids[view_name]: id2type[i['parent_id']] = None id2type[i['child_id']] = None topo = {i['child_id']: {i['parent_id']} for i in view2cr_ids[view_name]} leaf = list(set(toposort.toposort_flatten(topo)) - set([j for i in topo.values() for j in i])) leaf2show_types = {i: [t['child_id'] for t in CITypeRelation.get_by(parent_id=i)] for i in leaf} node2show_types = copy.deepcopy(leaf2show_types) def _find_parent(_node_id): parents = topo.get(_node_id, {}) for parent in parents: node2show_types.setdefault(parent, []).extend(node2show_types.get(_node_id, [])) _find_parent(parent) if not parents: return for l in leaf: _find_parent(l) for node_id in node2show_types: node2show_types[node_id] = [CITypeCache.get(i).to_dict() for i in set(node2show_types[node_id])] result[view_name] = dict(topo=list(map(list, toposort.toposort(topo))), topo_flatten=list(toposort.toposort_flatten(topo)), leaf=leaf, leaf2show_types=leaf2show_types, node2show_types=node2show_types, show_types=[CITypeCache.get(j).to_dict() for i in leaf2show_types.values() for j in i]) for type_id in id2type: id2type[type_id] = CITypeCache.get(type_id).to_dict() return result, id2type, sorted(name2id, key=lambda x: x[1])
def load(shuffle=False): # contains names like 'fullscreen', not 'porcupine.plugins.fullscreen' plugin_names = { name for finder, name, is_pkg in pkgutil.iter_modules(plugin_paths) if not name.startswith('_') } log.info("found %d plugins", len(plugin_names)) plugin_infos = {} # {name: (setup_before, setup_after, setup_func)} for name in plugin_names.copy(): log.debug("importing %s", name) try: module = importlib.import_module('rbtk.plugins.' + name) setup_before = set(getattr(module, 'setup_before', [])) setup_after = set(getattr(module, 'setup_after', [])) setup = module.setup except Exception: log.exception("problem with importing %s", name) continue # now we know that the plugin is ok, we can add its stuff to # dependencies and setup_funcs plugin_infos[name] = (setup_before, setup_after, setup) # setup_before and setup_after may contain names of plugins that are # not installed because they are for controlling the loading order, # not for requiring dependencies def valid_name(name): return name in plugin_infos dependencies = {name: set() for name in plugin_infos} for name, (setup_before, setup_after, setup) in plugin_infos.items(): dependencies[name].update(filter(valid_name, setup_after)) for reverse_dependency in filter(valid_name, setup_before): dependencies[reverse_dependency].add(name) # the toposort will partially work even if there's a circular # dependency, the CircularDependencyError is raised after doing # everything possible (see source code) loading_order = [] try: for names in map(list, toposort.toposort(dependencies)): if shuffle: random.shuffle(names) else: names.sort() loading_order.extend(names) except toposort.CircularDependencyError as e: parts = map("{} depends on {}".format, e.data.items()) log.error("circular dependency: %s", ', '.join(parts)) for name in loading_order: *junk, setup = plugin_infos[name] log.debug("running %s.setup()", name) try: setup() except Exception: log.exception("%s.setup() doesn't work", name)
def run_setup_functions(shuffle: bool) -> None: imported_infos = _get_successfully_imported_infos() # setup_before and setup_after may contain names of plugins that are not # installed because they are for controlling the loading order. That's fine # because toposort ignores missing things. dependencies: Dict[str, Set[str]] = {} for info in imported_infos: dependencies.setdefault(info.name, set()).update(info.setup_after) for reverse_dependency in info.setup_before: dependencies.setdefault(reverse_dependency, set()).add(info.name) # the toposort will partially work even if there's a circular # dependency, the CircularDependencyError is raised after doing # everything possible (see source code) loading_order: List[str] = [] try: # https://github.com/python/mypy/issues/9253 make_list: Callable[[Iterable[str]], List[str]] = list for names in map(make_list, toposort.toposort(dependencies)): if shuffle: # for plugin developers wanting to make sure that the # dependencies specified in setup_before and setup_after # are correct random.shuffle(names) else: # for consistency in UI (e.g. always same order of menu items) names.sort() loading_order.extend(names) loadable_infos = imported_infos except toposort.CircularDependencyError as e: log.exception("circular dependency") for info in imported_infos: if info.name in loading_order: loadable_infos.append(info) else: info.status = Status.CIRCULAR_DEPENDENCY_ERROR parts = ', '.join(f"{a} depends on {b}" for a, b in e.data.items()) info.error = f"Circular dependency error: {parts}" loadable_infos.sort(key=(lambda info: loading_order.index(info.name))) for info in loadable_infos: start = time.time() try: assert info.module is not None info.module.setup() except Exception: log.exception(f"{info.name}.setup() doesn't work") info.status = Status.SETUP_FAILED info.error = traceback.format_exc() else: info.status = Status.ACTIVE duration = time.time() - start log.debug("ran %s.setup() in %.3f milliseconds", info.name, duration * 1000)
def test_input_not_modified_when_cycle_error(self): data = {1: {2}, 2: {1}, 3: {4}, } orig = data.copy() self.assertRaises(ValueError, list, toposort(data)) self.assertEqual(data, orig)
def number_of_strongly_connected_components(adj, rev_adj): visited = set() cc = 0 order = toposort(rev_adj) for v in order: if v not in visited: explore(v, adj, visited, cc) cc += 1 return cc
def calculate_dependencies(): """Calculate test dependencies First do a topological sorting based on the dependencies. Then sort the different dependency groups based on priorities. """ order = [] for g in toposort(merge_dicts(dependencies, soft_dependencies)): for t in sorted(g, key=lambda x: (priorities[x], x)): order.append(t) return order
def test_input_not_modified(self): data = {2: {11}, 9: {11, 8}, 10: {11, 3}, 11: {7, 5}, 8: {7, 3, 8}, # includes something self-referential } orig = data.copy() results = list(toposort(data)) self.assertEqual(data, orig)
def ops_in_toposorted_order(ops): """Produces ops in deterministic order such that children are executed after parents""" graph_dict = tf_ops_to_graph(ops) toposorted = toposort.toposort(graph_dict) ops = [] # toposort assumes children are dependencies, reverse order for op_set in reversed(list(toposorted)): ops.extend(sorted(op_set, key=lambda op: op.name)) return ops
def test_strings(self): self.assertEqual(list(toposort({'2': {'11'}, '9': {'11', '8'}, '10': {'11', '3'}, '11': {'7', '5'}, '8': {'7', '3'}, })), [{'3', '5', '7'}, {'8', '11'}, {'2', '9', '10'}, ])
def calculate_dependencies(): """Calculate test dependencies First do a topological sorting based on the dependencies. Then sort the different dependency groups based on priorities. """ order = [] for group in toposort(dependencies): priority_sorted_group = sorted(group, key=lambda x: (priorities[x], x)) order.extend(priority_sorted_group) return order
def main(): CARS = 10 START_PTS = [] END_PTS = [] for i in xrange(CARS): START_PTS += [(100+i*30, 500, -math.pi/2)] START_PTS += [(100+i*30, 400, -math.pi/2)] END_PTS += [(370-i*30, 100, -math.pi/2)] END_PTS += [(370-i*30, 200, -math.pi/2)] dist_matrix = [] for start_pt in START_PTS: dist_vec = [] for end_pt in END_PTS: dist_vec.append(dubins.path_length(start_pt, end_pt, settings.TURNING_RADIUS)) print dist_vec dist_matrix.append(dist_vec) m = munkres.Munkres() indices = m.compute(dist_matrix) d_pt_match = {} for start_pt_ind, end_pt_ind in indices: d_pt_match[start_pt_ind] = end_pt_ind cars = [] for i in xrange(CARS): cars.append((START_PTS[i], END_PTS[d_pt_match[i]])) print cars[-1] COLLISION_DIST = 5 d_collisions = {} for i, (start_pt, end_pt) in enumerate(cars): d_collisions[i] = set() path, t = dubins.path_sample(start_pt, end_pt, settings.TURNING_RADIUS, 0.1) for j, (temp_start_pt, temp_end_pt) in enumerate(cars): if j == i: continue for pt in path: if norm(sub_vecs(pt, temp_end_pt)) < COLLISION_DIST: d_collisions[i].add(j) print d_collisions cars_order = list(toposort.toposort(d_collisions)) for car_set in cars_order[::-1]: print "Creating cars", car_set for car_ind in car_set: thread.start_new_thread(main_loop, ((cars[car_ind][0][0], cars[car_ind][0][1]), cars[car_ind][0][2], (cars[car_ind][1][0], cars[car_ind][1][1]), cars[car_ind][1][2])) time.sleep(10)
def ranks2order(ranks, majority = False): big = pd.DataFrame([[(ranks[j] > ranks[i]).sum() for i in ranks.columns] for j in ranks.columns]) big.columns = ranks.columns big.index = ranks.columns if not majority: majority = big.max().max() graph = sorted(big[big >= majority].stack().index.tolist()) big_dict = dict([(x, set(a[1] for a in y)) for (x, y) in itertools.groupby(graph, key=lambda x: x[0])]) true_order = list(toposort.toposort(big_dict)) print("Majority:", majority / len(ranks)) return true_order
def find_sync_groups(containers): deps = dict() name2container = dict() for container in containers: name2container[container.name] = container deps[container.name] = container.dependency_names() synch_groups = [] for synch_group_names in toposort.toposort(deps): synch_group = set() for container_name in synch_group_names: synch_group.add(name2container[container_name]) synch_groups.append(synch_group) return synch_groups
def build_groups(self): """Return topological sort of dependency groups, i.e, list of sets of containers which can be built in groups.""" result = [] for t in list(toposort.toposort(self.build_depends())): if t == set(['']): # We don't have to build the null dependency continue group = [] for tag in t: container_name = self.container_name_from_tag(tag) container = self.get_container(container_name) if container.buildable: group.append(container_name) result.append(group) return result
def _sort_modules(self): modules = {} for instance in self._instances: if not self._instances[instance]._disabled: _deps = self._instances[instance]._depends _afters = self._instances[instance]._after modules[instance] = _deps.union(_afters) try: return [m for m in toposort(modules) if m in self._instances] except ValueError as e: circular = findall('.*?\(\'?(\S+?)\'?,', e.args[0]) for instance in circular: self._instances[instance]._disabled = True debug(_('Circular dependencies found, disabled modules: %s') % ', '.join(circular)) return self._sort_modules()
def tf_toposort(ts, within_ops=None): all_ops = ge.get_forward_walk_ops([x.op for x in ts], within_ops=within_ops) deps = {} for op in all_ops: for o in op.outputs: deps[o] = set(op.inputs) sorted_ts = toposort(deps) # only keep the tensors from our original list ts_sorted_lists = [] for l in sorted_ts: keep = list(set(l).intersection(ts)) if keep: ts_sorted_lists.append(keep) return ts_sorted_lists
def tf_toposort(ts_inp, within_ops=None): """ Tensorflow topological sort """ all_ops = ge.get_forward_walk_ops([x.op for x in ts_inp], within_ops=within_ops) deps = {} for tf_op in all_ops: for outp in tf_op.outputs: deps[outp] = set(tf_op.inputs) sorted_ts = toposort(deps) # only keep the tensors from our original list ts_sorted_lists = [] for lst in sorted_ts: keep = list(set(lst).intersection(ts_inp)) if keep: ts_sorted_lists.append(keep) return ts_sorted_lists
def answer(): f = open('p079_keylog.txt', 'r') data = list(set(map(int, f.read().split()))) graph = {} for datum in data: datum = str(datum) if datum[2] in graph: graph[datum[2]] |= {datum[0], datum[1]} else: graph[datum[2]] = {datum[0], datum[1]} if datum[1] in graph: graph[datum[1]] |= {datum[0]} else: graph[datum[1]] = {datum[0]} graph = list(map(list, list(toposort(graph)))) graph = [item[0] for item in graph] return ''.join(graph)
def sort_migrations(migration_classes, flatten=False): migration_dependency_graph = { m.migration_id: set() if m.dependencies is None else set(m.dependencies) for m in migration_classes } migration_classes_by_id = { m.migration_id: m for m in migration_classes } migration_id_order = toposort(migration_dependency_graph) migration_execution_sets = [ {migration_classes_by_id[migration_id] for migration_id in execution_set} for execution_set in migration_id_order ] if flatten: return list(itertools.chain.from_iterable(migration_execution_sets)) else: return migration_execution_sets
def toposort_recipes(recipes): metadata = list(get_metadata(recipes)) name2recipe = defaultdict(list) for meta, recipe in zip(metadata, recipes): name2recipe[meta.get_value("package/name")].append(recipe) def get_inner_deps(dependencies): for dep in dependencies: name = dep.split()[0] if name in name2recipe: yield name dag = { meta.get_value("package/name"): set(get_inner_deps(chain(get_deps(meta), get_deps(meta, build=False)))) for meta in metadata } # note difference here from build-packages.py return toposort(dag)
def test_objects(self): o2 = object() o3 = object() o5 = object() o7 = object() o8 = object() o9 = object() o10 = object() o11 = object() self.assertEqual(list(toposort({o2: {o11}, o9: {o11, o8}, o10: {o11, o3}, o11: {o7, o5}, o8: {o7, o3, o8}, })), [{o3, o5, o7}, {o8, o11}, {o2, o9, o10}, ])
def test_sort_flatten(self): data = {2: {11}, 9: {11, 8}, 10: {11, 3}, 11: {7, 5}, 8: {7, 3, 8}, # includes something self-referential } expected = [{3, 5, 7}, {8, 11}, {2, 9, 10}] self.assertEqual(list(toposort(data)), expected) # now check the sorted results results = [] for item in expected: results.extend(sorted(item)) self.assertEqual(toposort_flatten(data), results) # and the unsorted results. break the results up into groups to compare them actual = toposort_flatten(data, False) results = [{i for i in actual[0:3]}, {i for i in actual[3:5]}, {i for i in actual[5:8]}] self.assertEqual(results, expected)
def resolve_dep_order(images_config): # Figure out what images are provided by this config # Anything not provided is assumed to exist already provided_images = dict() for image_name,image_config in images_config.items(): if container_system_option_names.isdisjoint(image_config.keys()): provided_images[image_name] = image_name else: for commit_spec in optional_plural(image_config, "commits"): dest = parse_dest_spec(commit_spec, "commits", image_name) provided_images[dest.image] = image_name for persist_spec in optional_plural(image_config, "persists"): dest = parse_dest_spec(commit_spec, "persists", image_name) provided_images[dest.image] = image_name # Figure out the images required (among those provided) by images in this config deps = dict() for image_name,image_config in images_config.items(): req = set() deps[image_name] = req if container_system_option_names.isdisjoint(image_config.keys()): from_name = image_config["from"].split(":", 1)[0] if from_name in provided_images: req.add(from_name) else: sys_config = nagoya.cli.cfg.read_one(image_config["system"]) for cont_config in sys_config.values(): image_name = cont_config["image"].split(":", 1)[0] if image_name in provided_images: req.add(image_name) # Toposort to sync groups, use original order of keys to order within groups image_names = [] for group in toposort.toposort(deps): image_names.extend(sorted(group, key=lambda n: images_config.keys().index(n))) return image_names
def run_annotators(self, document_chunk): for group in toposort(self.dependency_graph): current_results = [] for annotation_tag in group: annotator = self.annotators[annotation_tag] #task_d = [{dep: doc.annotations[dep] for dep in annotator.dependsOn()} for doc in document_chunk] task_data = document_chunk async_res = annotator.get_annotations_delay(task_data) current_results.append((annotation_tag, async_res)) # todo: we can save time by not waiting for all strategies, but only ones that are dependencies for annotation_tag, async_result in current_results: document_annotations = async_result.get() assert len(document_chunk) == len(document_annotations), "Annotator %s did not return the correct number of annotations!" if async_result.status != 'SUCCESS': ln.error("Annotator %s threw an error!", annotation_tag) if isinstance(async_result.result, Exception): raise async_result.result for annotation, document in zip(document_annotations, document_chunk): document.annotations[annotation_tag] = annotation return document_chunk
def test_toposort(): nodes = make_caterpillar_graph(length=2) graph = linearize.get_graph() print(list(toposort.toposort(graph)))