def testAlreadyPrioritized(self): graph = {1: set([2]), 2: set()} expected = deepcopy(graph) topo_prioritize(2, graph) self.assertEquals(graph, expected) sorted = toposort_flatten(graph) self.assertEqual(toposort_flatten(graph), [2, 1])
def topo_sort_removing_errors( graph: Dict[str, Set[str]], ) -> Tuple[List[str], Set[str]]: """ graph should be dict with values as sets: {'b': set('a'), 'c': set('b'), 'a': set()} That says b depends on a, and c depends on b, and a depends on nothing. The returned value is a tuple of form (['a', 'b', 'c'], set()) where the first item is the order to evaluate, and the second item is the list of bad dependencies. """ # toposort doesn't find self-dependencies, but they're a problem for us self_deps = {key for key, val in graph.items() if key in val} graph = {key: val for key, val in graph.items() if key not in self_deps} for maybe_bad_key, deps in graph.items(): if any(v not in graph for v in deps if v != "t"): # dependencies on t are OK return topo_sort_removing_errors({ key: val for key, val in graph.items() if key != maybe_bad_key }) try: return toposort_flatten(graph), set() except CircularDependencyError as e: circle = set(e.data.keys()) graph = {key: val for key, val in graph.items() if key not in circle} return toposort_flatten(graph), set(circle)
def test_sort_flatten(self): data = { 2: {11}, 9: {11, 8}, 10: {11, 3}, 11: {7, 5}, 8: {7, 3, 8}, # includes something self-referential } expected = [{3, 5, 7}, {8, 11}, {2, 9, 10}] self.assertEqual(list(toposort(data)), expected) # now check the sorted results results = [] for item in expected: results.extend(sorted(item)) self.assertEqual(toposort_flatten(data), results) # and the unsorted results. break the results up into groups to compare them actual = toposort_flatten(data, False) results = [{i for i in actual[0:3]}, {i for i in actual[3:5]}, {i for i in actual[5:8]}] self.assertEqual(results, expected)
def testAvoidCircularReference(self): graph = {1: set(), 2: set([1])} expected = deepcopy(graph) topo_prioritize(2, graph) self.assertEquals(graph, expected) sorted = toposort_flatten(graph) self.assertEqual(toposort_flatten(graph), [1, 2])
def get_relation_view(): _views = PreferenceRelationView.get_by(to_dict=True) views = [] if current_app.config.get("USE_ACL"): for i in _views: try: if ACLManager().has_permission(i.get('name'), ResourceTypeEnum.RELATION_VIEW, PermEnum.READ): views.append(i) except AbortException: pass else: views = _views view2cr_ids = dict() result = dict() name2id = list() for view in views: view2cr_ids.setdefault(view['name'], []).extend(json.loads(view['cr_ids'])) name2id.append([view['name'], view['id']]) id2type = dict() for view_name in view2cr_ids: for i in view2cr_ids[view_name]: id2type[i['parent_id']] = None id2type[i['child_id']] = None topo = {i['child_id']: {i['parent_id']} for i in view2cr_ids[view_name]} leaf = list(set(toposort.toposort_flatten(topo)) - set([j for i in topo.values() for j in i])) leaf2show_types = {i: [t['child_id'] for t in CITypeRelation.get_by(parent_id=i)] for i in leaf} node2show_types = copy.deepcopy(leaf2show_types) def _find_parent(_node_id): parents = topo.get(_node_id, {}) for parent in parents: node2show_types.setdefault(parent, []).extend(node2show_types.get(_node_id, [])) _find_parent(parent) if not parents: return for l in leaf: _find_parent(l) for node_id in node2show_types: node2show_types[node_id] = [CITypeCache.get(i).to_dict() for i in set(node2show_types[node_id])] result[view_name] = dict(topo=list(map(list, toposort.toposort(topo))), topo_flatten=list(toposort.toposort_flatten(topo)), leaf=leaf, leaf2show_types=leaf2show_types, node2show_types=node2show_types, show_types=[CITypeCache.get(j).to_dict() for i in leaf2show_types.values() for j in i]) for type_id in id2type: id2type[type_id] = CITypeCache.get(type_id).to_dict() return result, id2type, sorted(name2id, key=lambda x: x[1])
def testSimple(self): graph = {1: set(), 2: set()} expected = deepcopy(graph) expected[1].add(2) topo_prioritize(2, graph) self.assertEquals(graph, expected) sorted = toposort_flatten(graph) self.assertEqual(toposort_flatten(graph), [2, 1])
def evaluate(): data = request.get_json() logging.info("data sent for evaluation {}".format(data)) input_modules = data.get('modules') input_dependency = data.get('dependencyPairs') dep = {} for pairs in input_dependency: dependee = pairs['dependee'] dependentOn = pairs['dependentOn'] if dependee in input_modules: if dependee not in dep: dep[dependee] = {dependentOn} else: dep[dependee].add(dependentOn) for module in input_modules: if module not in dep: dep[module] = {module} try: result = toposort_flatten(dep, sort=True) except ValueError as e: modules = input_modules.copy() for mod in e.data.values(): for a in mod: if a in modules: modules.remove(a) for mod in e.data.keys(): if mod in modules: modules.remove(mod) input_modules = modules dep = {} for pairs in input_dependency: dependee = pairs['dependee'] dependentOn = pairs['dependentOn'] if dependee in input_modules: if dependee not in dep: dep[dependee] = {dependentOn} else: dep[dependee].add(dependentOn) for module in input_modules: if module not in dep: dep[module] = {module} result = toposort_flatten(dep, sort=True) logging.info("My result :{}".format(result)) return json.dumps(result)
def order_modules(self): runorder = None try: dependencies = self.collect_dependencies() runorder = toposort_flatten(dependencies) self.compute_reachability(dependencies) except ValueError: # cycle, try to break it then # if there's still a cycle, we cannot run the first cycle logger.info('Cycle in module dependencies, trying to drop optional fields') dependencies = self.collect_dependencies(only_required=True) runorder = toposort_flatten(dependencies) self.compute_reachability(dependencies) return runorder
def _process_inventory(self) -> typing.NoReturn: for dump_id in toposort.toposort_flatten(self._dependencies): if dump_id in self._processed: continue obj_type, schema, name = self._reverse_lookup[dump_id] LOGGER.debug('Processing %s %s.%s', obj_type, schema, name) for dep in self._dependencies[dump_id]: if dep not in self._processed: mt, ms, mn = self._reverse_lookup[dump_id] LOGGER.error( 'Dependency for %i (%s, %s, %s), ' '%i (%s, %s, %s) not processed', dump_id, obj_type, schema, name, dep, mt, ms, mn) raise RuntimeError if self._inventory[obj_type][schema][name].parent: self._process_child(obj_type, schema, name, dump_id) continue definition = self._inventory[obj_type][schema][name].definition entry = self._dump.add_entry( obj_type, schema, name, self._get_owner(definition), definition['sql'], dependencies=self._dependencies[dump_id], tablespace=definition.get('tablespace', ''), dump_id=dump_id) self._maybe_add_comment(entry, definition) self._processed.add(dump_id) self._objects += 1 self._process_sequence_set_owned_by()
def get_dependency_map( depman: 'DependencyManager', mods: LilacMods, ) -> Dict[str, Set['Dependency']]: map: Dict[str, Set['Dependency']] = defaultdict(set) shallow_map: Dict[str, Set[str]] = defaultdict(set) rmap: Dict[str, Set[str]] = defaultdict(set) for name, mod in mods.items(): depends = getattr(mod, 'repo_depends', ()) ds = [depman.get(d) for d in depends] if ds: for d in ds: shallow_map[name].add(d.pkgname) rmap[d.pkgname].add(name) map[name].update(ds) dep_order = toposort_flatten(shallow_map) for name in dep_order: if name in rmap: deps = map[name] dependers = rmap[name] for dd in dependers: map[dd].update(deps) return map
def ordered_task_instances_list(self): """ Start from a root task, and read task dependencies recursively. """ # NOTE consider Luiti modifies Luigi, same task instances maybe not unique. TODO Fix it def func(dep_dict, _curr_task): """ Generate a a dependencies graph. """ required_task_instances = _curr_task.requires() # 1. clean dep normal tasks as a list if not isinstance(required_task_instances, list): required_task_instances = [required_task_instances] dep_dict[_curr_task] = filter(lambda t1: bool(t1) and (not isinstance(t1, RootTask)), required_task_instances) # 2. selected undone tasks if self.check_output_recursive: dep_dict[_curr_task] = filter(lambda t1: not t1.output().exists(), dep_dict[_curr_task]) dep_dict[_curr_task] = set(dep_dict[_curr_task]) # 3. recursive for t1 in dep_dict[_curr_task]: func(dep_dict, t1) return dep_dict # 1. generate a dep dict task_instances_dep_dict = func(dict(), self.curr_task_intance) # 2. sort the DAG. from toposort import toposort_flatten ordered_task_instances_list = toposort_flatten(task_instances_dep_dict) return ordered_task_instances_list
def render_schema(self, **kwargs): "return full SQL schema built rendering indexed resources" # construct templating environment tplEnv = Environment(line_statement_prefix='--#') # build resource index rsrIdx = self.build_index() # iterates resource in 'topological' sorted order buf = [] for rsr in toposort_flatten(rsrIdx): # process resource template if any if rsr.tplstr: # add source informations to tplstr tplstr = self._fileHead % (self.get_rpath(rsr), rsr.tplstr) # compile template tpl = tplEnv.from_string(tplstr) # prepare rendering context ctx = {} ctx.update(rsr.tpldefaults) ctx.update(kwargs) # render template buf.append(tpl.render(ctx)) return u"\n".join(buf)
def mount_lowerdirs(self): # First, build a dependency graph in order to avoid duplicate entries dependencies = {} def dependency_path(dep): if 'image' in dep: return Image.download(dep['image'], dep['version']) if ':' in dep['imageName']: # It's an image return Image.download(dep['imageName']) else: # It's a container container = self.__class__(dep['imageName']) path = dep.get('path', str((container.path / 'overlay.fs').resolve())) return os.path.join(dep['imageName'], path) pending_deps = set(map(dependency_path, self.dependencies)) while len(pending_deps) > 0: path = pending_deps.pop() if isinstance(path, Image): path.extract() prev_layer = str(path.layers[-1].fs_path) dependencies[prev_layer] = set() for layer in reversed(path.layers[:-1]): dependencies[prev_layer] = {str(layer.fs_path)} prev_layer = str(layer.fs_path) else: name = path.split('/')[-2] if name not in dependencies: dependencies[path] = set(map(dependency_path, self.__class__(name).dependencies)) pending_deps |= dependencies[path] # Then sort it topologically. The list is reversed, because overlayfs # will check the mounts in order they are given, so the base fs has to # be the last one. dependencies = reversed(list(toposort_flatten(dependencies))) return (os.path.join(self.metadata_dir, dep) for dep in dependencies)
def refresh_provisioning_playbook(arg_vars, project_root): cluster_id = arg_vars['cluster_id'] tpl = util.provisioning_template() path = util.machine_profiles_path(project_root, cluster_id) profile_files = [f for f in listdir(path) if isfile(join(path, f))] services = {} service_graph = {} profiles = [] for f in profile_files: with open((path + "/" + f), "r") as handle: content = yaml.load(handle) profiles.append(content['profile_id']) for s in content.get('machine_services', []): rets = services.get(s, []) rets.append(content['profile_id']) services[s] = rets for s in services.keys(): with open((project_root + "/ansible/roles/" + s + "/defaults/main.yml"), "r") as text_file: content = yaml.load(text_file) service_graph[s] = set(content.get('service_dependencies', {})) service_seq = toposort_flatten(service_graph) with open(util.provisioning_file(project_root, cluster_id), "w") as text_file: text_file.write(tpl.render(cluster_id = cluster_id, services = services, profiles = profiles, service_seq = service_seq, service_graph = service_graph))
def find_joins_for_tables(joins, base_table, required_tables): """ Given a set of tables required for a slicer query, this function finds the joins required for the query and sorts them topologically. :return: A list of joins in the order that they must be joined to the query. :raises: MissingTableJoinException - If a table is required but there is no join for that table CircularJoinsException - If there is a circular dependency between two or more joins """ dependencies = defaultdict(set) slicer_joins = {join.table: join for join in joins} while required_tables: table = required_tables.pop() if table not in slicer_joins: raise MissingTableJoinException('Could not find a join for table {}' .format(str(table))) join = slicer_joins[table] tables_required_for_join = set(join.criterion.tables_) - {base_table, join.table} dependencies[join] |= {slicer_joins[table] for table in tables_required_for_join} required_tables += tables_required_for_join - {d.table for d in dependencies} try: return toposort_flatten(dependencies, sort=True) except CircularDependencyError as e: raise CircularJoinsException(str(e))
def apply(self, model): if len(model.graph.node) == 1: # single-node graph, nothing to sort return (model, False) # Gather graph structure graph_dependencies = {} node_list = [n for n in model.graph.node ] # I also need the list to remove the nodes for node_idx, n in enumerate(node_list): node_pred = model.find_direct_predecessors(n) if node_pred is None: # Will also eliminate nodes that are floating around for some reason continue node_dependencies = [node_list.index(pred) for pred in node_pred] graph_dependencies[node_idx] = set(node_dependencies) # Sort nodes sorted_node_indexes = toposort_flatten(graph_dependencies) # Remove nodes and insert them in order # Can't remove nodes before if I want to use model.find_direct_predecessors() for n in node_list: model.graph.node.remove(n) for new_idx, sorted_idx in enumerate(sorted_node_indexes): model.graph.node.insert(new_idx, node_list[sorted_idx]) return (model, False)
def expand(self, states): seen = set() depends = defaultdict(list) queue = deque() for state in states: queue.append(state) seen.add(state) while len(queue): state = queue.popleft() for arc in self.get_arcs(state, EPSILON): depends[arc[1]].append((arc[0], arc[3])) if arc[1] in seen: continue queue.append(arc[1]) seen.add(arc[1]) depends_for_toposort = { key: {state for state, weight in value} for key, value in depends.items() } order = toposort_flatten(depends_for_toposort) next_states = states for next_state in order: next_states[next_state] = self.combine_weights( *([next_states.get(next_state)] + [ next_states[prev_state] + weight for prev_state, weight in depends[next_state] ])) return next_states
def sorted_models(self, app_config): dep_dict, dep_class_map = self.get_model_dependencies(app_config.get_models()) try: return [dep_class_map[x] for x in toposort_flatten(dep_dict)] except ValueError as ex: raise SeederCommandError(str(ex))
def expand(self, states): seen = set() depends = defaultdict(list) queue = deque() for state in states: queue.append(state) seen.add(state) while len(queue): state = queue.popleft() for arc in self.get_arcs(state, EPSILON): depends[arc[1]].append((arc[0], arc[3])) if arc[1] in seen: continue queue.append(arc[1]) seen.add(arc[1]) depends_for_toposort = {key: {state for state, weight in value} for key, value in depends.items()} order = toposort_flatten(depends_for_toposort) next_states = states for next_state in order: next_states[next_state] = self.combine_weights( *([next_states.get(next_state)] + [next_states[prev_state] + weight for prev_state, weight in depends[next_state]])) return next_states
def sort_dependencies(cls, apps='all'): """ Return a list of models in their order of dependency """ dependencies = defaultdict(set) model_objects = [] if apps != 'all': apps = apps.split(',') for c in get_subclasses(ViewedModel): if apps != 'all' and c._meta.app_label not in apps: continue class_string = model_default_table_name(c) for dependency in getattr(c, 'dependencies', []): m = get_model(app_name=dependency[0], model_name=dependency[1]) dependencies[class_string].add(model_default_table_name(m)) flattened = toposort_flatten(dependencies) print(flattened) for name in flattened: app, model = name.split('_') if app == 'None' or model == 'viewedmodel': continue model_object = get_model(app, model) if hasattr(model_object, 'sql'): model_objects.append(model_object) return model_objects
def toposort_flatten_checking(d, sort=False): '''Essentially just toposort_flatten, but with checking for self-referential dependencies and defaulting to sort=False to preserves order when possible''' for k, v in d.iteritems(): assert k not in v, 'Self-referencing dependency: {}'.format(k) return toposort.toposort_flatten(d, sort=sort)
def show(self): today = datetime.datetime.now().date() result = toposort_flatten(self._dependencies) gantt_tasks = {} for id_ in result: gantt_tasks[id_] = task = gantt.Task( name=self._tasks[id_][0], start=today, duration=self._tasks[id_][1], depends_of=[ gantt_tasks[blocker] for blocker in self._dependencies[id_] ], #color=self.COLORS.get(self._tasks[id_][2], self.COLORS[()]), resources=list(self._resources.values()), ) self._project.add_task(task) directory = tempfile.gettempdir() self._project.make_svg_for_resources( filename=os.path.join(directory, 'gantt_resources.svg'), today=today, resources=list(self._resources.values()), ) self._project.make_svg_for_tasks( filename=os.path.join(directory, 'gantt_tasks.svg'), today=today, )
def step(self, name, deps, f, *args, **kwargs): """ Function to add a step to the computational graph. deps [String]: a list of nodes in the graph that the function depends on f [Function]: node to execute args [List]: list of arguments to be passed into the function """ # create a new stair for this function stair = Stair(name, deps, f, *args, **kwargs) # update our computational graph with this stair self.stairs[name] = stair # check that we have the dependencies in the, and assume any # that aren't are inputs for dep in deps: # first check that the variable isn't already in the graph if dep in self.stairs: continue # create an input stair and enter it into our graph input_stair = Stair(dep, [], lambda v: v) self.stairs[dep] = input_stair self.graph[input_stair] = set([]) # add our dependencie to the graph self.graph[stair] = set([self.stairs[dep] for dep in deps]) # ensure that our new graph compiles self.slist = toposort_flatten(self.graph) # return self for chaining return self
def find_dependencies( metadatas_pb: 'Mapping[str, G.ArchivePayload]', existing_package_ids: 'Collection[str]') \ -> 'ArchiveDependencyResult': """ Return a topologically-sorted list of dependencies for the package IDs. :param metadatas_pb: The collection of gRPC ArchivePayload to re-order, indexed by package ID. :param existing_package_ids: Collection of package IDs that are to be treated as pre-existing. :return: A topologically-sorted dictionary of package IDs to ArchivePayload objects. Iterations through the dictionary are guaranteed to be in the correct order. """ dependencies = defaultdict(set) for package_id, archive_payload in metadatas_pb.items(): for module_pb in archive_payload.daml_lf_1.modules: for data_type_pb in module_pb.data_types: deps = None if data_type_pb.HasField('record'): deps = find_dependencies_of_fwts( data_type_pb.record.fields) elif data_type_pb.HasField('variant'): deps = find_dependencies_of_fwts( data_type_pb.variant.fields) if deps is not None: deps.difference_update(existing_package_ids) dependencies[package_id].update(deps) # identify any completely missing dependencies # TODO: This doesn't handle transitive missing dependencies; this will be a problem once # DAML has proper dependency support unresolvable_package_ids = [] for package_id, package_dependencies in dependencies.items(): if not package_dependencies.issubset(metadatas_pb): unresolvable_package_ids.append(package_id) m_pb = {} sorted_package_ids = toposort_flatten(dependencies) for package_id in sorted_package_ids: if package_id not in unresolvable_package_ids: required_package = metadatas_pb.get(package_id) if required_package is not None: m_pb[package_id] = required_package else: LOG.warning('Failed to find a package %r', package_id) if unresolvable_package_ids: # This isn't a major problem in the grand scheme of things because the caller continually # retries all unknown packages, so if the missing dependent packages are eventually # uploaded, we will end up back in this function and all of the packages will be parsed. LOG.warning( 'Some package IDs could not be resolved, so packages that depend on these IDs ' 'will be unavailable: %r', unresolvable_package_ids) return ArchiveDependencyResult(sorted_archives=m_pb, unresolvable_archives={ pkg_id: metadatas_pb[pkg_id] for pkg_id in unresolvable_package_ids })
def _order_arguments(cls, arguments: List[Argument], location) -> List[Argument]: dag = {} # Build dag to have deps specified by depends_on, and a dep # chain through all the NON-depends_on arguments in order. for arg in arguments: arg._all_arguments = arguments try: if arg.depends_on: dag[arg] = set( cls._get_argument_by_name(arguments, n) for n in arg.depends_on) else: dag[arg] = set() except LookupError as e: parse_requires(False, f"Unknown argument name: {e.args[0]}", loc=arg.location) # Compute an order which honors the deps try: return toposort_flatten(dag, sort=True) except CircularDependencyError: parse_requires(False, "The dependencies between arguments are cyclic.", loc=location)
def __init__(self, fp: TextIO, cflags: str = None, work_dir: Optional[str] = None): """Construct Program object from a json file. Args: fp: TextIO of json object. cflags: The cflags send to HLS. work_dir: Specifiy a working directory as a string. If None, a temporary one will be created. """ obj = json.load(fp) self.top: str = obj['top'] self.cflags = cflags self.headers: Dict[str, str] = obj.get('headers', {}) if work_dir is None: self.work_dir = tempfile.mkdtemp(prefix='tapa-') self.is_temp = True else: self.work_dir = os.path.abspath(work_dir) os.makedirs(self.work_dir, exist_ok=True) self.is_temp = False self.toplevel_ports = tuple(map(Port, obj['tasks'][self.top]['ports'])) self._tasks: Dict[str, Task] = collections.OrderedDict( (name, Task(name=name, **obj['tasks'][name])) for name in toposort.toposort_flatten( {k: set(v.get('tasks', ())) for k, v in obj['tasks'].items()})) self.frt_interface = obj['tasks'][self.top].get('frt_interface') self.tcl_files: Dict[str, str] = {}
def find_joins_for_tables(joins, base_table, required_tables): """ Given a set of tables required for a dataset query, this function finds the joins required for the query and sorts them topologically. :return: A list of joins in the order that they must be joined to the query. :raises: MissingTableJoinException - If a table is required but there is no join for that table CircularJoinsException - If there is a circular dependency between two or more joins """ dependencies = defaultdict(set) slicer_joins = {join.table: join for join in joins} while required_tables: table = required_tables.pop() if table not in slicer_joins: raise MissingTableJoinException("Could not find a join for table {}".format(str(table))) join = slicer_joins[table] tables_required_for_join = set(join.criterion.tables_) - { base_table, join.table, } dependencies[join] |= {slicer_joins[table] for table in tables_required_for_join} required_tables += tables_required_for_join - {d.table for d in dependencies} try: return toposort_flatten(dependencies, sort=True) except CircularDependencyError as e: raise CircularJoinsException(str(e))
def get_dependency_map( depman: DependencyManager, mods: LilacMods, ) -> Dict[str, Set[Dependency]]: map: Dict[str, Set[Dependency]] = defaultdict(set) shallow_map: Dict[str, Set[str]] = defaultdict(set) rmap: Dict[str, Set[str]] = defaultdict(set) for name, mod in mods.items(): depends = getattr(mod, 'repo_depends', ()) ds = [depman.get(d) for d in depends] if ds: for d in ds: shallow_map[name].add(d.pkgname) rmap[d.pkgname].add(name) map[name].update(ds) dep_order = toposort_flatten(shallow_map) for name in dep_order: if name in rmap: deps = map[name] dependers = rmap[name] for dd in dependers: map[dd].update(deps) return map
def verify_graph(self, nodes): # The generated layered graphs add dependencies randomly to modules within each layer. # Because of that, we cannot always specify a fixed expected list of nodes without making # tests indeterministic. Instead, we verify topological sorting by re-creating the same graph, # sorting it topologicaly and assert the two lists to be the same. graph = {n: set(n.deps) for n in nodes} self.assertEqual(toposort_flatten(graph), nodes)
def init_def_files(chemin): "" depends_on = {} Lexer_pp.def_file_raw_d = {} Lexer_pp.def_file_exp_d = {} assert os.path.exists(chemin) for root, dirs, files in os.walk(chemin): for fn in files: if fn.endswith('.def'): fp = os.path.join(root, fn) print('*******************************************') print(fp) fd = open(fp) prg = fd.read() fd.close() lexer.input(prg) for t in lexer: pass assert fn not in Lexer_pp.def_file_raw_d Lexer_pp.def_file_raw_d[fn] = lexer.definition_d depends_on[fn] = { k for k, v in lexer.definition_d.items() if v == (None, None) } def_file_l = toposort_flatten(depends_on) for df in def_file_l: Lexer_pp.def_file_exp_d[df] = d = Lexer_pp.def_file_raw_d[df].copy() Lexer_pp._expand_definitions(d)
def _apply_traits(self, pipeline_def): transformers = [ trait.transformer() for trait in pipeline_def._traits_dict.values() ] transformers_dict = {t.name: t for t in transformers} transformer_names = set(transformers_dict.keys()) for transformer in transformers: if not set(transformer.dependencies()).issubset(transformer_names): missing = set(transformer.dependencies()) - transformer_names raise ModelValidationError( f'{pipeline_def}: trait requires missing traits: ' + ', '.join(missing)) # order transformers according to dependencies transformer_dependencies = { t.name: t.order_dependencies() & transformer_names for t in transformers } ordered_transformers = [] for name in toposort.toposort_flatten(transformer_dependencies): ordered_transformers.append(transformers_dict[name]) # inject new steps for transformer in ordered_transformers: for step in transformer.inject_steps(): pipeline_def.add_step(step) # do remaining processing for transformer in ordered_transformers: transformer.process_pipeline_args(pipeline_def)
def get_sorted_features(available_features: Iterable[Feature] = None): """ Register default features and setuptools entrypoint 'ddb_features' inside features registry. Features are registered in order for their dependency to be registered first with a topological sort. Withing a command phase, actions are executed in the order of their feature registration. """ if available_features is None: available_features = _available_features entrypoint_features = {f.name: f for f in available_features} for entry_point in pkg_resources.iter_entry_points('ddb_features'): feature = entry_point.load()() entrypoint_features[feature.name] = feature required_dependencies, toposort_data = _prepare_dependencies_data( entrypoint_features) _check_missing_dependencies(entrypoint_features, required_dependencies) dependencies = config.data.get('dependencies') if dependencies: for feat, feat_dependencies in dependencies.items(): if feat not in toposort_data: toposort_data[feat] = set() for feat_dependency in feat_dependencies: toposort_data[feat].add(feat_dependency) sorted_feature_names = toposort_flatten(toposort_data, sort=True) for feature_name in sorted_feature_names: feature = entrypoint_features.get(feature_name) if feature: yield feature
def test_on_lean_version(version, version_history): print(f'\nRunning tests on Lean version {version}') key = remote_ref_from_lean_version(version) mathlib_prev = version_history[key]['mathlib']['latest_test'] \ if key in version_history and 'mathlib' in version_history[key] else None version_projects = [p for p in projects if version in projects[p].branches] print(f'version projects: {version_projects}') if not changes_on_version(version, version_projects, version_history): print( f'no projects have changed on version {version} since the last run.\n' ) return if version in projects['mathlib'].branches: add_success_to_version_history(version, 'mathlib', version_history) ordered_projects = toposort_flatten( {p: projects[p].dependencies for p in version_projects}) # if 'mathlib' in ordered_projects: # ordered_projects.remove('mathlib') failures = {} i = 0 while i < len(ordered_projects): p = ordered_projects[i] missing_deps = [ dep for dep in projects[p].dependencies if dep not in ordered_projects ] if p not in version_projects or len(missing_deps) > 0: print(f'removing {p}') del ordered_projects[i] if p in version_projects: is_new = not previous_run_exists_and_failed( version, p, version_history) failing_test( version, p, version_history, failures, DependencyFailure(p, version, is_new, missing_deps)) else: i += 1 if len(ordered_projects) > 0 and any(project != 'mathlib' for project in ordered_projects): print(f'\nbuilding projects in order: {ordered_projects}') if 'mathlib' in ordered_projects: update_mathlib_to_version(version) for project_name in [ project_name for project_name in ordered_projects if project_name != 'mathlib' ]: test_project_on_version(version, project_name, failures, version_history) if len(failures) > 0: print(f'\n{len(failures)} failures:') for f in failures: print(failures[f]) failures[f].report_issue(version_history, mathlib_prev)
def main(): os.environ['WASMER_PUBLISH_SCRIPT_IS_RUNNING'] = '1' parser = argparse.ArgumentParser( description='Publish the Wasmer crates to crates.io') parser.add_argument( '--no-dry-run', default=False, action='store_true', help='Run the script without actually publishing anything to crates.io' ) args = vars(parser.parse_args()) global no_dry_run no_dry_run = args['no_dry_run'] # get the order to publish the crates in order = list(toposort_flatten(dep_graph, sort=True)) for crate in order: print("Publishing `{}`...".format(crate)) if not is_crate_already_published(crate): publish_crate(crate) else: print("`{}` was already published!".format(crate)) continue # sleep for 16 seconds between crates to ensure the crates.io index has time to update # this can be optimized with knowledge of our dep graph via toposort; we can even publish # crates in parallel; however this is out of scope for the first version of this script if no_dry_run: print( "Sleeping for 16 seconds to allow the `crates.io` index to update..." ) time.sleep(16) else: print("In dry-run: not sleeping for crates.io to update.")
def _apply_traits(self, pipeline_def): transformers = [ trait.transformer() for trait in pipeline_def._traits_dict.values() ] transformers_dict = {t.name: t for t in transformers} transformer_names = set(transformers_dict.keys()) # order transformers according to dependencies transformer_dependencies = { t.name: t.dependencies() & transformer_names for t in transformers } ordered_transformers = [] for name in toposort.toposort_flatten(transformer_dependencies): ordered_transformers.append(transformers_dict[name]) # inject new steps for transformer in ordered_transformers: for step in transformer.inject_steps(): pipeline_def.add_step(step) # do remaining processing for transformer in ordered_transformers: transformer.process_pipeline_args(pipeline_def)
def gen_layered_graph(layer_count, nodes_per_layer, deps_per_node=5): """Generates a module dependency graph that has `layer_count` layers, with each module only depending on a random selection of the modules below it""" def node(f_layer, f_node): return ModuleNode('MockLib{}_{}'.format(f_layer, f_node), ModuleNode.LIBRARY) layers = [[node(l, n) for n in xrange(nodes_per_layer)] for l in xrange(layer_count)] app_node = ModuleNode('App', ModuleNode.APP, layers[0]) node_graph = {} for i, layer in enumerate(layers): lower_layers = layers[(i + 1):] if i != len(layers) - 1 else [] lower_merged = merge_lists(lower_layers) for node in layer: if deps_per_node < len(lower_merged): node.deps = random.sample(lower_merged, deps_per_node) else: node.deps = lower_merged node_graph[node] = set(node.deps) return app_node, (toposort_flatten(node_graph) + [app_node])
def sorted_rules(self, rules_cfg): """ Rules sorted according to rule dependencies. :type rules_cfg: dict :arg rules_cfg: rules as specified in logic-engine configuration :rtype: list :return: Rules to be evaluated by the rule engine. """ initial_rules = {name: Rule(name, cfg) for name, cfg in rules_cfg.items()} dependencies = {} try: for name, rule in initial_rules.items(): dependencies[name] = set() required_facts = rule.expr.required_names for fact in required_facts: rule_for_fact = self.rule_for(fact) if rule_for_fact: dependencies[name].add(rule_for_fact) ordered_dependencies = toposort_flatten(dependencies) except Exception: # pragma: no cover self.logger.exception("Unexpected error!") raise self.logger.debug(f"Calculated the following order for evaluating rules:\n{ordered_dependencies}") return [initial_rules[rule] for rule in ordered_dependencies]
def filter_get_services(project): result = {} for service in project['services']: result[service] = set(project['services'][service].get( 'depends_on', [])) result = toposort.toposort_flatten(result) return result
def sort_dependent_tables(table_defs, base_path, illegal_tables, schema, empty_tables): deps_dict = {} for table_def in table_defs: table_schema = table_def.find('table-schema') if table_schema.text.lower() != schema: continue table_name = table_def.find("table-name") # print(table_name.text) disposed = table_def.find("disposed") if disposed.text != "true": deps_dict.update({ table_name.text.lower(): get_table_deps(table_name, table_def, deps_dict, empty_tables, illegal_tables, schema) }) deps_list = toposort_flatten(deps_dict) # for dep in deps_list: # print(dep) return deps_list
def ordered_task_instances_list(self): """ Start from a root task, and read task dependencies recursively. """ # NOTE consider Luiti modifies Luigi, same task instances maybe not unique. TODO Fix it def func(dep_dict, _curr_task): """ Generate a a dependencies graph. """ required_task_instances = _curr_task.requires() if not isinstance(required_task_instances, list): required_task_instances = [required_task_instances] dep_dict[_curr_task] = filter( lambda t1: bool(t1) and (not isinstance(t1, RootTask)), required_task_instances) if self.check_output_recursive: dep_dict[_curr_task] = filter(lambda t1: t1.output().exists(), dep_dict[_curr_task]) dep_dict[_curr_task] = set(dep_dict[_curr_task]) for t1 in dep_dict[_curr_task]: func(dep_dict, t1) return dep_dict # 1. generate a dep dict task_instances_dep_dict = func(dict(), self.curr_task_intance) # 2. sort the DAG. from toposort import toposort_flatten ordered_task_instances_list = toposort_flatten(task_instances_dep_dict) return ordered_task_instances_list
def recreateSequence(darcs): from toposort import toposort_flatten dependFrom={} for u,v in darcs: dependFrom[v]=set([u]) return toposort_flatten(dependFrom)
def resolveDependencies(self): import toposort #build dependancy graph dg = self.dependancyGraph() #solve the dependency tree return toposort.toposort_flatten(dg)
def process_spec(specs): """Builds mapping class -> set of all descendants.""" tree = dict((name, set(children)) for name, children, _ in specs) attr_map = dict((name, attrs) for name, _, attrs in specs) for node in toposort_flatten(tree): children = tree[node] for child in set(children): tree[node] |= tree[child] return tree, attr_map
def create_base_query(self, sql_generator): self.anchor_table = self.get_data_table(self.PATIENTS).alias(self.BASE_PREFIX + self.PATIENTS) anchor_column = self.anchor_table.c.get(QDMConstants.PATIENT_ID_COL).label(self.outer_colname(QDMConstants.PATIENT_ID_COL)) base_from = self.anchor_table base_select_cols = [anchor_column] so_list = list(toposort_flatten(self.raw_symbol_table.so_temporal_dependencies)) self.merge_so_lists(so_list, self.data_criteria_names_used) so_done = [] for var in self.variables.values(): if not var.is_specific_occurrence_target(): sel = var.get_selectable(self, correlate=False) query = sel.get_selectable() sel.set_subquery_table(sql_generator.create_table_from_select(var.get_short_name(), query)) query = sel.get_selectable() for colname in [QDMConstants.UNIQUE_ID, QDMConstants.STARTTIME, QDMConstants.ENDTIME]: col = sel.get_column(colname) if col is not None: base_select_cols.append(col) base_from = base_from.join(query, sel.get_column(QDMConstants.PATIENT_ID_COL) == anchor_column, isouter=True) for var in self.specific_occurrence_targets.values(): sel = var.get_selectable(self, correlate=False) query = sel.get_selectable() sel.set_subquery_table(sql_generator.create_table_from_select(var.get_short_name(), query)) processed_sos = dict() for var in self.specific_occurrences.values(): sel = var.get_selectable(self, correlate=False) root = var.get_raw_id().get_root() join_conds = [sel.get_column(QDMConstants.PATIENT_ID_COL) == anchor_column] prior_sos = processed_sos.get(root) if prior_sos == None: processed_sos[root] = [sel] else: for prior_so in prior_sos: join_conds.append(sel.get_column(QDMConstants.UNIQUE_ID) != prior_so.get_column(QDMConstants.UNIQUE_ID)) prior_sos.append(sel) base_from = base_from.join(sel.get_from(), and_(*join_conds), isouter=True) for colname in [QDMConstants.UNIQUE_ID, QDMConstants.STARTTIME, QDMConstants.ENDTIME]: col = sel.get_column(colname) if col is not None: base_select_cols.append(col.label(var.to_so_colname(col.name))) # base_select = select(base_select_cols).select_from(base_from) # self.base_select = base_select.alias('patient_base') # self.anchor_column = self.base_select.corresponding_column(anchor_column) self.base_select = sql_generator.create_table_from_select('patient_base', select(base_select_cols).select_from(base_from)) for var in self.specific_occurrences.values(): for colname in [QDMConstants.UNIQUE_ID, QDMConstants.STARTTIME, QDMConstants.ENDTIME]: var.get_selectable(self).set_column(colname, self.base_select.c.get(var.to_so_colname(colname))) self.anchor_column = self.base_select.c.get(self.outer_colname(QDMConstants.PATIENT_ID_COL)) self.anchor_table = self.base_select
def sortCodes(self): """put self.codes in a working evaluation order""" codeFromOutput = dict((c.outName, c) for c in self.codes) deps = {} for c in self.codes: outName = c.outName inNames = c.possibleVars.intersection(codeFromOutput.keys()) inNames.discard(outName) deps[outName] = inNames self.codes = [codeFromOutput[n] for n in toposort.toposort_flatten(deps)]
def testMiddle(self): # graph = {1: set(), 2: set([3]), 3: set()} expected = deepcopy(graph) expected[1].add(2) topo_prioritize(2, graph) self.assertEquals(graph, expected) self.assertEqual(toposort_flatten(graph), [3, 2, 1])
def test_sort_flatten(self): data = {2: {11}, 9: {11, 8}, 10: {11, 3}, 11: {7, 5}, 8: {7, 3, 8}, # includes something self-referential } expected = [{3, 5, 7}, {8, 11}, {2, 9, 10}] self.assertEqual(list(toposort(data)), expected) # now check the sorted results results = [] for item in expected: results.extend(sorted(item)) self.assertEqual(toposort_flatten(data), results) # and the unsorted results. break the results up into groups to compare them actual = toposort_flatten(data, False) results = [{i for i in actual[0:3]}, {i for i in actual[3:5]}, {i for i in actual[5:8]}] self.assertEqual(results, expected)
def toposort_send(signal, sender, **kwargs): """Send signal in topological order to all connected receivers.""" from toposort import toposort_flatten if not signal.receivers: return [] else: return [(receiver, receiver(sender, **kwargs)) for receiver in toposort_flatten({ receiver: toposort_extract(receiver) for receiver in signal.receivers_for(sender) })]
def __init__(self, thread_id, name, experiment, component_id, max_results, cache_results): threading.Thread.__init__(self) self.threadID = thread_id self.name = name self.experiment = experiment self.comp_id = component_id self.result = {} self.max_results = max_results self.cache_results = cache_results print "Submitting topology to storm. End component", self.comp_id exp = Experiment.objects.get(pk=self.experiment) graph = exp.workflow.graph_data graph_data = {} print graph tmp = graph.split(',') for elem in tmp: first_node = elem.split(":")[0] second_node = elem.split(":")[1] if second_node in graph_data: depend_nodes = graph_data[second_node] depend_nodes.add(first_node) else: graph_data[second_node] = set() graph_data[second_node].add(first_node) topological_graph = toposort_flatten(graph_data) print "Graph after topological sort", topological_graph message = { 'exp_id': self.experiment, 'result': self.comp_id, 'graph': topological_graph, 'components': defaultdict()} for data in topological_graph: component_id = int(data) comp = Component.objects.get(pk=component_id) if comp.operation_type.function_type == 'Create': if comp.operation_type.function_arg == 'Table': filename = comp.operation_type.function_subtype_arg input_data = read_csv(filename) message['input'] = {} for elem in list(input_data.columns): message['input'][elem] = list(input_data[elem]) message['cols'] = list(input_data.columns) # message['input'] = input_data.to_dict() serialized_obj = serializers.serialize('json', [comp.operation_type, ]) print "Component_id", component_id, " ", comp.operation_type message['components'][data] = serialized_obj print "Message ", message r = redis.StrictRedis(host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=0) self.pubsub = r.pubsub(ignore_subscribe_messages=True) self.pubsub.subscribe("Exp " + str(self.experiment)) ret = r.publish('workflow', json.dumps(message)) print "return", ret
def get_parentmap_toposort(self, reverse=False): """ Returns an iterator in topological order over element, parent pairs. """ parent_map = self.get_parentmap() digraph = graph.digraph(parent_map.items()) if reverse: digraph = graph.reverse(digraph) for element in toposort_flatten(digraph, sort=False): yield element, parent_map.get(element, None)
def sort(self, queryset): """Sort given the queryset in a topological order. :param queryset: django queryset :return: sorted list of `Migration` objects """ first_migration = queryset.first() uids = {migration.uid: migration for migration in (self.filter( case__ci_project=first_migration.case.ci_project, ) if first_migration else self.all())} mapping = { migration.uid: {migration.parent.uid} if migration.parent else set() for migration in queryset} return [uids[uid] for uid in toposort_flatten(mapping)]
def sort(data): deps = {} output = {} for app in data['assets']: if app.get('depends'): deps[app['name']] = set(d['name'] for d in app['depends']) else: deps[app['name']] = set() output[app['name']] = app #import pprint #pprint.pprint(toposort_flatten(deps)) #assert False, 'asdf' return toposort_flatten(deps), output
def _sort_features(features): graph = {} for f in features: graph[f.name] = set(f.deps) order = toposort_flatten(graph, sort=True) ret = [] for name in order: for f in features: if f.name == name: ret.append(f) return ret
def mount_lowerdirs(self): # First, build a dependency graph in order to avoid duplicate entries dependencies = {} pending_deps = set(self.dependencies) while len(pending_deps) > 0: name = pending_deps.pop() if name not in dependencies: dependencies[name] = set(Container(name).dependencies) pending_deps |= dependencies[name] # Then sort it topologically. The list is reversed, because overlayfs # will check the mounts in order they are given, so the base fs has to # be the last one. dependencies = reversed(list(toposort_flatten(dependencies))) return ':'.join('/var/lib/machines/{}/overlay.fs'.format(dep) for dep in dependencies)
def assignLevel(self): if not self.changed: return dependencyDict = {} for node in self.nodes: dependencyDict[node] = node.precursors self.maxLevel = 0 self.sortedNodes = toposort_flatten(dependencyDict) for node in self.sortedNodes: maxLevel = -1 for precursor in node.precursors: maxLevel = max(maxLevel, node.level) node.level = maxLevel + 1 self.maxLevel = max(self.maxLevel,node.level)
def get_order(setl_graph): nodes = collections.defaultdict(set) for typ in [setl.Extract, setl.Transform, setl.Load]: for task in setl_graph.subjects(RDF.type, typ): task = setl_graph.resource(task) for used in task[prov.used]: nodes[task.identifier].add(used.identifier) for usage in task[prov.qualifiedUsage]: used = usage.value(prov.entity) nodes[task.identifier].add(used.identifier) for generated in task.subjects(prov.wasGeneratedBy): nodes[generated.identifier].add(task.identifier) return toposort_flatten(nodes)
def ____topologicSortModules(self,path,modules): dependencyDict = {} for module in modules: dependencySet = set() fModule = open(os.path.join(path,module),'r') for line in fModule: line = line.strip() if line.startswith('include'): dependency = line.replace('include','').strip() dependencySet.add(dependency[1:-1]) fModule.close() dependencyDict[module] = dependencySet from toposort import toposort_flatten modules = toposort_flatten(dependencyDict) return modules