def testAlreadyPrioritized(self):
graph = {1: set([2]), 2: set()}
expected = deepcopy(graph)
topo_prioritize(2, graph)
self.assertEquals(graph, expected)
sorted = toposort_flatten(graph)
self.assertEqual(toposort_flatten(graph), [2, 1])
def topo_sort_removing_errors(
graph: Dict[str, Set[str]], ) -> Tuple[List[str], Set[str]]:
"""
graph should be dict with values as sets: {'b': set('a'), 'c': set('b'), 'a': set()}
That says b depends on a, and c depends on b, and a depends on nothing.
The returned value is a tuple of form (['a', 'b', 'c'], set()) where the first item
is the order to evaluate, and the second item is the list of bad dependencies.
"""
# toposort doesn't find self-dependencies, but they're a problem for us
self_deps = {key for key, val in graph.items() if key in val}
graph = {key: val for key, val in graph.items() if key not in self_deps}
for maybe_bad_key, deps in graph.items():
if any(v not in graph for v in deps
if v != "t"): # dependencies on t are OK
return topo_sort_removing_errors({
key: val
for key, val in graph.items() if key != maybe_bad_key
})
try:
return toposort_flatten(graph), set()
except CircularDependencyError as e:
circle = set(e.data.keys())
graph = {key: val for key, val in graph.items() if key not in circle}
return toposort_flatten(graph), set(circle)
def test_sort_flatten(self):
data = {
2: {11},
9: {11, 8},
10: {11, 3},
11: {7, 5},
8: {7, 3, 8}, # includes something self-referential
}
expected = [{3, 5, 7}, {8, 11}, {2, 9, 10}]
self.assertEqual(list(toposort(data)), expected)
# now check the sorted results
results = []
for item in expected:
results.extend(sorted(item))
self.assertEqual(toposort_flatten(data), results)
# and the unsorted results. break the results up into groups to compare them
actual = toposort_flatten(data, False)
results = [{i
for i in actual[0:3]}, {i
for i in actual[3:5]},
{i
for i in actual[5:8]}]
self.assertEqual(results, expected)
def testAvoidCircularReference(self):
graph = {1: set(), 2: set([1])}
expected = deepcopy(graph)
topo_prioritize(2, graph)
self.assertEquals(graph, expected)
sorted = toposort_flatten(graph)
self.assertEqual(toposort_flatten(graph), [1, 2])
def get_relation_view():
_views = PreferenceRelationView.get_by(to_dict=True)
views = []
if current_app.config.get("USE_ACL"):
for i in _views:
try:
if ACLManager().has_permission(i.get('name'),
ResourceTypeEnum.RELATION_VIEW,
PermEnum.READ):
views.append(i)
except AbortException:
pass
else:
views = _views
view2cr_ids = dict()
result = dict()
name2id = list()
for view in views:
view2cr_ids.setdefault(view['name'], []).extend(json.loads(view['cr_ids']))
name2id.append([view['name'], view['id']])
id2type = dict()
for view_name in view2cr_ids:
for i in view2cr_ids[view_name]:
id2type[i['parent_id']] = None
id2type[i['child_id']] = None
topo = {i['child_id']: {i['parent_id']} for i in view2cr_ids[view_name]}
leaf = list(set(toposort.toposort_flatten(topo)) - set([j for i in topo.values() for j in i]))
leaf2show_types = {i: [t['child_id'] for t in CITypeRelation.get_by(parent_id=i)] for i in leaf}
node2show_types = copy.deepcopy(leaf2show_types)
def _find_parent(_node_id):
parents = topo.get(_node_id, {})
for parent in parents:
node2show_types.setdefault(parent, []).extend(node2show_types.get(_node_id, []))
_find_parent(parent)
if not parents:
return
for l in leaf:
_find_parent(l)
for node_id in node2show_types:
node2show_types[node_id] = [CITypeCache.get(i).to_dict() for i in set(node2show_types[node_id])]
result[view_name] = dict(topo=list(map(list, toposort.toposort(topo))),
topo_flatten=list(toposort.toposort_flatten(topo)),
leaf=leaf,
leaf2show_types=leaf2show_types,
node2show_types=node2show_types,
show_types=[CITypeCache.get(j).to_dict()
for i in leaf2show_types.values() for j in i])
for type_id in id2type:
id2type[type_id] = CITypeCache.get(type_id).to_dict()
return result, id2type, sorted(name2id, key=lambda x: x[1])
def testAvoidCircularReference(self):
graph = {1: set(),
2: set([1])}
expected = deepcopy(graph)
topo_prioritize(2, graph)
self.assertEquals(graph, expected)
sorted = toposort_flatten(graph)
self.assertEqual(toposort_flatten(graph), [1, 2])
def testAlreadyPrioritized(self):
graph = {1: set([2]),
2: set()}
expected = deepcopy(graph)
topo_prioritize(2, graph)
self.assertEquals(graph, expected)
sorted = toposort_flatten(graph)
self.assertEqual(toposort_flatten(graph), [2, 1])
def testSimple(self):
graph = {1: set(), 2: set()}
expected = deepcopy(graph)
expected[1].add(2)
topo_prioritize(2, graph)
self.assertEquals(graph, expected)
sorted = toposort_flatten(graph)
self.assertEqual(toposort_flatten(graph), [2, 1])
def testSimple(self):
graph = {1: set(),
2: set()}
expected = deepcopy(graph)
expected[1].add(2)
topo_prioritize(2, graph)
self.assertEquals(graph, expected)
sorted = toposort_flatten(graph)
self.assertEqual(toposort_flatten(graph), [2, 1])
def evaluate():
data = request.get_json()
logging.info("data sent for evaluation {}".format(data))
input_modules = data.get('modules')
input_dependency = data.get('dependencyPairs')
dep = {}
for pairs in input_dependency:
dependee = pairs['dependee']
dependentOn = pairs['dependentOn']
if dependee in input_modules:
if dependee not in dep:
dep[dependee] = {dependentOn}
else:
dep[dependee].add(dependentOn)
for module in input_modules:
if module not in dep:
dep[module] = {module}
try:
result = toposort_flatten(dep, sort=True)
except ValueError as e:
modules = input_modules.copy()
for mod in e.data.values():
for a in mod:
if a in modules:
modules.remove(a)
for mod in e.data.keys():
if mod in modules:
modules.remove(mod)
input_modules = modules
dep = {}
for pairs in input_dependency:
dependee = pairs['dependee']
dependentOn = pairs['dependentOn']
if dependee in input_modules:
if dependee not in dep:
dep[dependee] = {dependentOn}
else:
dep[dependee].add(dependentOn)
for module in input_modules:
if module not in dep:
dep[module] = {module}
result = toposort_flatten(dep, sort=True)
logging.info("My result :{}".format(result))
return json.dumps(result)
def order_modules(self):
runorder = None
try:
dependencies = self.collect_dependencies()
runorder = toposort_flatten(dependencies)
self.compute_reachability(dependencies)
except ValueError: # cycle, try to break it then
# if there's still a cycle, we cannot run the first cycle
logger.info('Cycle in module dependencies, trying to drop optional fields')
dependencies = self.collect_dependencies(only_required=True)
runorder = toposort_flatten(dependencies)
self.compute_reachability(dependencies)
return runorder
def _process_inventory(self) -> typing.NoReturn:
for dump_id in toposort.toposort_flatten(self._dependencies):
if dump_id in self._processed:
continue
obj_type, schema, name = self._reverse_lookup[dump_id]
LOGGER.debug('Processing %s %s.%s', obj_type, schema, name)
for dep in self._dependencies[dump_id]:
if dep not in self._processed:
mt, ms, mn = self._reverse_lookup[dump_id]
LOGGER.error(
'Dependency for %i (%s, %s, %s), '
'%i (%s, %s, %s) not processed', dump_id, obj_type,
schema, name, dep, mt, ms, mn)
raise RuntimeError
if self._inventory[obj_type][schema][name].parent:
self._process_child(obj_type, schema, name, dump_id)
continue
definition = self._inventory[obj_type][schema][name].definition
entry = self._dump.add_entry(
obj_type,
schema,
name,
self._get_owner(definition),
definition['sql'],
dependencies=self._dependencies[dump_id],
tablespace=definition.get('tablespace', ''),
dump_id=dump_id)
self._maybe_add_comment(entry, definition)
self._processed.add(dump_id)
self._objects += 1
self._process_sequence_set_owned_by()
def get_dependency_map(
depman: 'DependencyManager', mods: LilacMods,
) -> Dict[str, Set['Dependency']]:
map: Dict[str, Set['Dependency']] = defaultdict(set)
shallow_map: Dict[str, Set[str]] = defaultdict(set)
rmap: Dict[str, Set[str]] = defaultdict(set)
for name, mod in mods.items():
depends = getattr(mod, 'repo_depends', ())
ds = [depman.get(d) for d in depends]
if ds:
for d in ds:
shallow_map[name].add(d.pkgname)
rmap[d.pkgname].add(name)
map[name].update(ds)
dep_order = toposort_flatten(shallow_map)
for name in dep_order:
if name in rmap:
deps = map[name]
dependers = rmap[name]
for dd in dependers:
map[dd].update(deps)
return map
def ordered_task_instances_list(self):
"""
Start from a root task, and read task dependencies recursively.
"""
# NOTE consider Luiti modifies Luigi, same task instances maybe not unique. TODO Fix it
def func(dep_dict, _curr_task):
""" Generate a a dependencies graph. """
required_task_instances = _curr_task.requires()
# 1. clean dep normal tasks as a list
if not isinstance(required_task_instances, list):
required_task_instances = [required_task_instances]
dep_dict[_curr_task] = filter(lambda t1: bool(t1) and (not isinstance(t1, RootTask)), required_task_instances)
# 2. selected undone tasks
if self.check_output_recursive:
dep_dict[_curr_task] = filter(lambda t1: not t1.output().exists(), dep_dict[_curr_task])
dep_dict[_curr_task] = set(dep_dict[_curr_task])
# 3. recursive
for t1 in dep_dict[_curr_task]:
func(dep_dict, t1)
return dep_dict
# 1. generate a dep dict
task_instances_dep_dict = func(dict(), self.curr_task_intance)
# 2. sort the DAG.
from toposort import toposort_flatten
ordered_task_instances_list = toposort_flatten(task_instances_dep_dict)
return ordered_task_instances_list
def render_schema(self, **kwargs):
"return full SQL schema built rendering indexed resources"
# construct templating environment
tplEnv = Environment(line_statement_prefix='--#')
# build resource index
rsrIdx = self.build_index()
# iterates resource in 'topological' sorted order
buf = []
for rsr in toposort_flatten(rsrIdx):
# process resource template if any
if rsr.tplstr:
# add source informations to tplstr
tplstr = self._fileHead % (self.get_rpath(rsr), rsr.tplstr)
# compile template
tpl = tplEnv.from_string(tplstr)
# prepare rendering context
ctx = {}
ctx.update(rsr.tpldefaults)
ctx.update(kwargs)
# render template
buf.append(tpl.render(ctx))
return u"\n".join(buf)
def mount_lowerdirs(self):
# First, build a dependency graph in order to avoid duplicate entries
dependencies = {}
def dependency_path(dep):
if 'image' in dep:
return Image.download(dep['image'], dep['version'])
if ':' in dep['imageName']: # It's an image
return Image.download(dep['imageName'])
else: # It's a container
container = self.__class__(dep['imageName'])
path = dep.get('path', str((container.path / 'overlay.fs').resolve()))
return os.path.join(dep['imageName'], path)
pending_deps = set(map(dependency_path, self.dependencies))
while len(pending_deps) > 0:
path = pending_deps.pop()
if isinstance(path, Image):
path.extract()
prev_layer = str(path.layers[-1].fs_path)
dependencies[prev_layer] = set()
for layer in reversed(path.layers[:-1]):
dependencies[prev_layer] = {str(layer.fs_path)}
prev_layer = str(layer.fs_path)
else:
name = path.split('/')[-2]
if name not in dependencies:
dependencies[path] = set(map(dependency_path, self.__class__(name).dependencies))
pending_deps |= dependencies[path]
# Then sort it topologically. The list is reversed, because overlayfs
# will check the mounts in order they are given, so the base fs has to
# be the last one.
dependencies = reversed(list(toposort_flatten(dependencies)))
return (os.path.join(self.metadata_dir, dep) for dep in dependencies)
def refresh_provisioning_playbook(arg_vars, project_root):
cluster_id = arg_vars['cluster_id']
tpl = util.provisioning_template()
path = util.machine_profiles_path(project_root, cluster_id)
profile_files = [f for f in listdir(path) if isfile(join(path, f))]
services = {}
service_graph = {}
profiles = []
for f in profile_files:
with open((path + "/" + f), "r") as handle:
content = yaml.load(handle)
profiles.append(content['profile_id'])
for s in content.get('machine_services', []):
rets = services.get(s, [])
rets.append(content['profile_id'])
services[s] = rets
for s in services.keys():
with open((project_root + "/ansible/roles/" + s + "/defaults/main.yml"), "r") as text_file:
content = yaml.load(text_file)
service_graph[s] = set(content.get('service_dependencies', {}))
service_seq = toposort_flatten(service_graph)
with open(util.provisioning_file(project_root, cluster_id), "w") as text_file:
text_file.write(tpl.render(cluster_id = cluster_id,
services = services,
profiles = profiles,
service_seq = service_seq,
service_graph = service_graph))
def find_joins_for_tables(joins, base_table, required_tables):
"""
Given a set of tables required for a slicer query, this function finds the joins required for the query and
sorts them topologically.
:return:
A list of joins in the order that they must be joined to the query.
:raises:
MissingTableJoinException - If a table is required but there is no join for that table
CircularJoinsException - If there is a circular dependency between two or more joins
"""
dependencies = defaultdict(set)
slicer_joins = {join.table: join
for join in joins}
while required_tables:
table = required_tables.pop()
if table not in slicer_joins:
raise MissingTableJoinException('Could not find a join for table {}'
.format(str(table)))
join = slicer_joins[table]
tables_required_for_join = set(join.criterion.tables_) - {base_table, join.table}
dependencies[join] |= {slicer_joins[table]
for table in tables_required_for_join}
required_tables += tables_required_for_join - {d.table for d in dependencies}
try:
return toposort_flatten(dependencies, sort=True)
except CircularDependencyError as e:
raise CircularJoinsException(str(e))
def apply(self, model):
if len(model.graph.node) == 1:
# single-node graph, nothing to sort
return (model, False)
# Gather graph structure
graph_dependencies = {}
node_list = [n for n in model.graph.node
] # I also need the list to remove the nodes
for node_idx, n in enumerate(node_list):
node_pred = model.find_direct_predecessors(n)
if node_pred is None:
# Will also eliminate nodes that are floating around for some reason
continue
node_dependencies = [node_list.index(pred) for pred in node_pred]
graph_dependencies[node_idx] = set(node_dependencies)
# Sort nodes
sorted_node_indexes = toposort_flatten(graph_dependencies)
# Remove nodes and insert them in order
# Can't remove nodes before if I want to use model.find_direct_predecessors()
for n in node_list:
model.graph.node.remove(n)
for new_idx, sorted_idx in enumerate(sorted_node_indexes):
model.graph.node.insert(new_idx, node_list[sorted_idx])
return (model, False)
def expand(self, states):
seen = set()
depends = defaultdict(list)
queue = deque()
for state in states:
queue.append(state)
seen.add(state)
while len(queue):
state = queue.popleft()
for arc in self.get_arcs(state, EPSILON):
depends[arc[1]].append((arc[0], arc[3]))
if arc[1] in seen:
continue
queue.append(arc[1])
seen.add(arc[1])
depends_for_toposort = {
key: {state
for state, weight in value}
for key, value in depends.items()
}
order = toposort_flatten(depends_for_toposort)
next_states = states
for next_state in order:
next_states[next_state] = self.combine_weights(
*([next_states.get(next_state)] + [
next_states[prev_state] + weight
for prev_state, weight in depends[next_state]
]))
return next_states
def sorted_models(self, app_config):
dep_dict, dep_class_map = self.get_model_dependencies(app_config.get_models())
try:
return [dep_class_map[x] for x in toposort_flatten(dep_dict)]
except ValueError as ex:
raise SeederCommandError(str(ex))
def expand(self, states):
seen = set()
depends = defaultdict(list)
queue = deque()
for state in states:
queue.append(state)
seen.add(state)
while len(queue):
state = queue.popleft()
for arc in self.get_arcs(state, EPSILON):
depends[arc[1]].append((arc[0], arc[3]))
if arc[1] in seen:
continue
queue.append(arc[1])
seen.add(arc[1])
depends_for_toposort = {key: {state for state, weight in value}
for key, value in depends.items()}
order = toposort_flatten(depends_for_toposort)
next_states = states
for next_state in order:
next_states[next_state] = self.combine_weights(
*([next_states.get(next_state)] +
[next_states[prev_state] + weight
for prev_state, weight in depends[next_state]]))
return next_states
def sort_dependencies(cls, apps='all'):
"""
Return a list of models in their order of dependency
"""
dependencies = defaultdict(set)
model_objects = []
if apps != 'all':
apps = apps.split(',')
for c in get_subclasses(ViewedModel):
if apps != 'all' and c._meta.app_label not in apps:
continue
class_string = model_default_table_name(c)
for dependency in getattr(c, 'dependencies', []):
m = get_model(app_name=dependency[0], model_name=dependency[1])
dependencies[class_string].add(model_default_table_name(m))
flattened = toposort_flatten(dependencies)
print(flattened)
for name in flattened:
app, model = name.split('_')
if app == 'None' or model == 'viewedmodel':
continue
model_object = get_model(app, model)
if hasattr(model_object, 'sql'):
model_objects.append(model_object)
return model_objects
def toposort_flatten_checking(d, sort=False):
'''Essentially just toposort_flatten, but with checking for
self-referential dependencies and defaulting to sort=False
to preserves order when possible'''
for k, v in d.iteritems():
assert k not in v, 'Self-referencing dependency: {}'.format(k)
return toposort.toposort_flatten(d, sort=sort)
def show(self):
today = datetime.datetime.now().date()
result = toposort_flatten(self._dependencies)
gantt_tasks = {}
for id_ in result:
gantt_tasks[id_] = task = gantt.Task(
name=self._tasks[id_][0],
start=today,
duration=self._tasks[id_][1],
depends_of=[
gantt_tasks[blocker] for blocker in self._dependencies[id_]
],
#color=self.COLORS.get(self._tasks[id_][2], self.COLORS[()]),
resources=list(self._resources.values()),
)
self._project.add_task(task)
directory = tempfile.gettempdir()
self._project.make_svg_for_resources(
filename=os.path.join(directory, 'gantt_resources.svg'),
today=today,
resources=list(self._resources.values()),
)
self._project.make_svg_for_tasks(
filename=os.path.join(directory, 'gantt_tasks.svg'),
today=today,
)
def step(self, name, deps, f, *args, **kwargs):
"""
Function to add a step to the computational graph.
deps [String]: a list of nodes in the graph that the function depends on
f [Function]: node to execute
args [List]: list of arguments to be passed into the function
"""
# create a new stair for this function
stair = Stair(name, deps, f, *args, **kwargs)
# update our computational graph with this stair
self.stairs[name] = stair
# check that we have the dependencies in the, and assume any
# that aren't are inputs
for dep in deps:
# first check that the variable isn't already in the graph
if dep in self.stairs: continue
# create an input stair and enter it into our graph
input_stair = Stair(dep, [], lambda v: v)
self.stairs[dep] = input_stair
self.graph[input_stair] = set([])
# add our dependencie to the graph
self.graph[stair] = set([self.stairs[dep] for dep in deps])
# ensure that our new graph compiles
self.slist = toposort_flatten(self.graph)
# return self for chaining
return self
def find_dependencies(
metadatas_pb: 'Mapping[str, G.ArchivePayload]', existing_package_ids: 'Collection[str]') \
-> 'ArchiveDependencyResult':
"""
Return a topologically-sorted list of dependencies for the package IDs.
:param metadatas_pb:
The collection of gRPC ArchivePayload to re-order, indexed by package ID.
:param existing_package_ids:
Collection of package IDs that are to be treated as pre-existing.
:return:
A topologically-sorted dictionary of package IDs to ArchivePayload objects. Iterations
through the dictionary are guaranteed to be in the correct order.
"""
dependencies = defaultdict(set)
for package_id, archive_payload in metadatas_pb.items():
for module_pb in archive_payload.daml_lf_1.modules:
for data_type_pb in module_pb.data_types:
deps = None
if data_type_pb.HasField('record'):
deps = find_dependencies_of_fwts(
data_type_pb.record.fields)
elif data_type_pb.HasField('variant'):
deps = find_dependencies_of_fwts(
data_type_pb.variant.fields)
if deps is not None:
deps.difference_update(existing_package_ids)
dependencies[package_id].update(deps)
# identify any completely missing dependencies
# TODO: This doesn't handle transitive missing dependencies; this will be a problem once
# DAML has proper dependency support
unresolvable_package_ids = []
for package_id, package_dependencies in dependencies.items():
if not package_dependencies.issubset(metadatas_pb):
unresolvable_package_ids.append(package_id)
m_pb = {}
sorted_package_ids = toposort_flatten(dependencies)
for package_id in sorted_package_ids:
if package_id not in unresolvable_package_ids:
required_package = metadatas_pb.get(package_id)
if required_package is not None:
m_pb[package_id] = required_package
else:
LOG.warning('Failed to find a package %r', package_id)
if unresolvable_package_ids:
# This isn't a major problem in the grand scheme of things because the caller continually
# retries all unknown packages, so if the missing dependent packages are eventually
# uploaded, we will end up back in this function and all of the packages will be parsed.
LOG.warning(
'Some package IDs could not be resolved, so packages that depend on these IDs '
'will be unavailable: %r', unresolvable_package_ids)
return ArchiveDependencyResult(sorted_archives=m_pb,
unresolvable_archives={
pkg_id: metadatas_pb[pkg_id]
for pkg_id in unresolvable_package_ids
})
def _order_arguments(cls, arguments: List[Argument],
location) -> List[Argument]:
dag = {}
# Build dag to have deps specified by depends_on, and a dep
# chain through all the NON-depends_on arguments in order.
for arg in arguments:
arg._all_arguments = arguments
try:
if arg.depends_on:
dag[arg] = set(
cls._get_argument_by_name(arguments, n)
for n in arg.depends_on)
else:
dag[arg] = set()
except LookupError as e:
parse_requires(False,
f"Unknown argument name: {e.args[0]}",
loc=arg.location)
# Compute an order which honors the deps
try:
return toposort_flatten(dag, sort=True)
except CircularDependencyError:
parse_requires(False,
"The dependencies between arguments are cyclic.",
loc=location)
def __init__(self,
fp: TextIO,
cflags: str = None,
work_dir: Optional[str] = None):
"""Construct Program object from a json file.
Args:
fp: TextIO of json object.
cflags: The cflags send to HLS.
work_dir: Specifiy a working directory as a string. If None, a temporary
one will be created.
"""
obj = json.load(fp)
self.top: str = obj['top']
self.cflags = cflags
self.headers: Dict[str, str] = obj.get('headers', {})
if work_dir is None:
self.work_dir = tempfile.mkdtemp(prefix='tapa-')
self.is_temp = True
else:
self.work_dir = os.path.abspath(work_dir)
os.makedirs(self.work_dir, exist_ok=True)
self.is_temp = False
self.toplevel_ports = tuple(map(Port, obj['tasks'][self.top]['ports']))
self._tasks: Dict[str, Task] = collections.OrderedDict(
(name, Task(name=name, **obj['tasks'][name]))
for name in toposort.toposort_flatten(
{k: set(v.get('tasks', ()))
for k, v in obj['tasks'].items()}))
self.frt_interface = obj['tasks'][self.top].get('frt_interface')
self.tcl_files: Dict[str, str] = {}
def find_joins_for_tables(joins, base_table, required_tables):
"""
Given a set of tables required for a dataset query, this function finds the joins required for the query and
sorts them topologically.
:return:
A list of joins in the order that they must be joined to the query.
:raises:
MissingTableJoinException - If a table is required but there is no join for that table
CircularJoinsException - If there is a circular dependency between two or more joins
"""
dependencies = defaultdict(set)
slicer_joins = {join.table: join for join in joins}
while required_tables:
table = required_tables.pop()
if table not in slicer_joins:
raise MissingTableJoinException("Could not find a join for table {}".format(str(table)))
join = slicer_joins[table]
tables_required_for_join = set(join.criterion.tables_) - {
base_table,
join.table,
}
dependencies[join] |= {slicer_joins[table] for table in tables_required_for_join}
required_tables += tables_required_for_join - {d.table for d in dependencies}
try:
return toposort_flatten(dependencies, sort=True)
except CircularDependencyError as e:
raise CircularJoinsException(str(e))
def get_dependency_map(
depman: DependencyManager,
mods: LilacMods,
) -> Dict[str, Set[Dependency]]:
map: Dict[str, Set[Dependency]] = defaultdict(set)
shallow_map: Dict[str, Set[str]] = defaultdict(set)
rmap: Dict[str, Set[str]] = defaultdict(set)
for name, mod in mods.items():
depends = getattr(mod, 'repo_depends', ())
ds = [depman.get(d) for d in depends]
if ds:
for d in ds:
shallow_map[name].add(d.pkgname)
rmap[d.pkgname].add(name)
map[name].update(ds)
dep_order = toposort_flatten(shallow_map)
for name in dep_order:
if name in rmap:
deps = map[name]
dependers = rmap[name]
for dd in dependers:
map[dd].update(deps)
return map
def verify_graph(self, nodes):
# The generated layered graphs add dependencies randomly to modules within each layer.
# Because of that, we cannot always specify a fixed expected list of nodes without making
# tests indeterministic. Instead, we verify topological sorting by re-creating the same graph,
# sorting it topologicaly and assert the two lists to be the same.
graph = {n: set(n.deps) for n in nodes}
self.assertEqual(toposort_flatten(graph), nodes)
def init_def_files(chemin):
""
depends_on = {}
Lexer_pp.def_file_raw_d = {}
Lexer_pp.def_file_exp_d = {}
assert os.path.exists(chemin)
for root, dirs, files in os.walk(chemin):
for fn in files:
if fn.endswith('.def'):
fp = os.path.join(root, fn)
print('*******************************************')
print(fp)
fd = open(fp)
prg = fd.read()
fd.close()
lexer.input(prg)
for t in lexer:
pass
assert fn not in Lexer_pp.def_file_raw_d
Lexer_pp.def_file_raw_d[fn] = lexer.definition_d
depends_on[fn] = {
k
for k, v in lexer.definition_d.items() if v == (None, None)
}
def_file_l = toposort_flatten(depends_on)
for df in def_file_l:
Lexer_pp.def_file_exp_d[df] = d = Lexer_pp.def_file_raw_d[df].copy()
Lexer_pp._expand_definitions(d)
def _apply_traits(self, pipeline_def):
transformers = [
trait.transformer()
for trait in pipeline_def._traits_dict.values()
]
transformers_dict = {t.name: t for t in transformers}
transformer_names = set(transformers_dict.keys())
for transformer in transformers:
if not set(transformer.dependencies()).issubset(transformer_names):
missing = set(transformer.dependencies()) - transformer_names
raise ModelValidationError(
f'{pipeline_def}: trait requires missing traits: ' +
', '.join(missing))
# order transformers according to dependencies
transformer_dependencies = {
t.name: t.order_dependencies() & transformer_names
for t in transformers
}
ordered_transformers = []
for name in toposort.toposort_flatten(transformer_dependencies):
ordered_transformers.append(transformers_dict[name])
# inject new steps
for transformer in ordered_transformers:
for step in transformer.inject_steps():
pipeline_def.add_step(step)
# do remaining processing
for transformer in ordered_transformers:
transformer.process_pipeline_args(pipeline_def)
def get_sorted_features(available_features: Iterable[Feature] = None):
"""
Register default features and setuptools entrypoint 'ddb_features' inside features registry.
Features are registered in order for their dependency to be registered first with a topological sort.
Withing a command phase, actions are executed in the order of their feature registration.
"""
if available_features is None:
available_features = _available_features
entrypoint_features = {f.name: f for f in available_features}
for entry_point in pkg_resources.iter_entry_points('ddb_features'):
feature = entry_point.load()()
entrypoint_features[feature.name] = feature
required_dependencies, toposort_data = _prepare_dependencies_data(
entrypoint_features)
_check_missing_dependencies(entrypoint_features, required_dependencies)
dependencies = config.data.get('dependencies')
if dependencies:
for feat, feat_dependencies in dependencies.items():
if feat not in toposort_data:
toposort_data[feat] = set()
for feat_dependency in feat_dependencies:
toposort_data[feat].add(feat_dependency)
sorted_feature_names = toposort_flatten(toposort_data, sort=True)
for feature_name in sorted_feature_names:
feature = entrypoint_features.get(feature_name)
if feature:
yield feature
def test_on_lean_version(version, version_history):
print(f'\nRunning tests on Lean version {version}')
key = remote_ref_from_lean_version(version)
mathlib_prev = version_history[key]['mathlib']['latest_test'] \
if key in version_history and 'mathlib' in version_history[key] else None
version_projects = [p for p in projects if version in projects[p].branches]
print(f'version projects: {version_projects}')
if not changes_on_version(version, version_projects, version_history):
print(
f'no projects have changed on version {version} since the last run.\n'
)
return
if version in projects['mathlib'].branches:
add_success_to_version_history(version, 'mathlib', version_history)
ordered_projects = toposort_flatten(
{p: projects[p].dependencies
for p in version_projects})
# if 'mathlib' in ordered_projects:
# ordered_projects.remove('mathlib')
failures = {}
i = 0
while i < len(ordered_projects):
p = ordered_projects[i]
missing_deps = [
dep for dep in projects[p].dependencies
if dep not in ordered_projects
]
if p not in version_projects or len(missing_deps) > 0:
print(f'removing {p}')
del ordered_projects[i]
if p in version_projects:
is_new = not previous_run_exists_and_failed(
version, p, version_history)
failing_test(
version, p, version_history, failures,
DependencyFailure(p, version, is_new, missing_deps))
else:
i += 1
if len(ordered_projects) > 0 and any(project != 'mathlib'
for project in ordered_projects):
print(f'\nbuilding projects in order: {ordered_projects}')
if 'mathlib' in ordered_projects:
update_mathlib_to_version(version)
for project_name in [
project_name for project_name in ordered_projects
if project_name != 'mathlib'
]:
test_project_on_version(version, project_name, failures,
version_history)
if len(failures) > 0:
print(f'\n{len(failures)} failures:')
for f in failures:
print(failures[f])
failures[f].report_issue(version_history, mathlib_prev)
def main():
os.environ['WASMER_PUBLISH_SCRIPT_IS_RUNNING'] = '1'
parser = argparse.ArgumentParser(
description='Publish the Wasmer crates to crates.io')
parser.add_argument(
'--no-dry-run',
default=False,
action='store_true',
help='Run the script without actually publishing anything to crates.io'
)
args = vars(parser.parse_args())
global no_dry_run
no_dry_run = args['no_dry_run']
# get the order to publish the crates in
order = list(toposort_flatten(dep_graph, sort=True))
for crate in order:
print("Publishing `{}`...".format(crate))
if not is_crate_already_published(crate):
publish_crate(crate)
else:
print("`{}` was already published!".format(crate))
continue
# sleep for 16 seconds between crates to ensure the crates.io index has time to update
# this can be optimized with knowledge of our dep graph via toposort; we can even publish
# crates in parallel; however this is out of scope for the first version of this script
if no_dry_run:
print(
"Sleeping for 16 seconds to allow the `crates.io` index to update..."
)
time.sleep(16)
else:
print("In dry-run: not sleeping for crates.io to update.")
def _apply_traits(self, pipeline_def):
transformers = [
trait.transformer()
for trait in pipeline_def._traits_dict.values()
]
transformers_dict = {t.name: t for t in transformers}
transformer_names = set(transformers_dict.keys())
# order transformers according to dependencies
transformer_dependencies = {
t.name: t.dependencies() & transformer_names
for t in transformers
}
ordered_transformers = []
for name in toposort.toposort_flatten(transformer_dependencies):
ordered_transformers.append(transformers_dict[name])
# inject new steps
for transformer in ordered_transformers:
for step in transformer.inject_steps():
pipeline_def.add_step(step)
# do remaining processing
for transformer in ordered_transformers:
transformer.process_pipeline_args(pipeline_def)
def gen_layered_graph(layer_count, nodes_per_layer, deps_per_node=5):
"""Generates a module dependency graph that has `layer_count` layers,
with each module only depending on a random selection of the modules
below it"""
def node(f_layer, f_node):
return ModuleNode('MockLib{}_{}'.format(f_layer, f_node),
ModuleNode.LIBRARY)
layers = [[node(l, n) for n in xrange(nodes_per_layer)]
for l in xrange(layer_count)]
app_node = ModuleNode('App', ModuleNode.APP, layers[0])
node_graph = {}
for i, layer in enumerate(layers):
lower_layers = layers[(i + 1):] if i != len(layers) - 1 else []
lower_merged = merge_lists(lower_layers)
for node in layer:
if deps_per_node < len(lower_merged):
node.deps = random.sample(lower_merged, deps_per_node)
else:
node.deps = lower_merged
node_graph[node] = set(node.deps)
return app_node, (toposort_flatten(node_graph) + [app_node])
def sorted_rules(self, rules_cfg):
"""
Rules sorted according to rule dependencies.
:type rules_cfg: dict
:arg rules_cfg: rules as specified in logic-engine configuration
:rtype: list
:return: Rules to be evaluated by the rule engine.
"""
initial_rules = {name: Rule(name, cfg) for name, cfg in rules_cfg.items()}
dependencies = {}
try:
for name, rule in initial_rules.items():
dependencies[name] = set()
required_facts = rule.expr.required_names
for fact in required_facts:
rule_for_fact = self.rule_for(fact)
if rule_for_fact:
dependencies[name].add(rule_for_fact)
ordered_dependencies = toposort_flatten(dependencies)
except Exception: # pragma: no cover
self.logger.exception("Unexpected error!")
raise
self.logger.debug(f"Calculated the following order for evaluating rules:\n{ordered_dependencies}")
return [initial_rules[rule] for rule in ordered_dependencies]
def filter_get_services(project):
result = {}
for service in project['services']:
result[service] = set(project['services'][service].get(
'depends_on', []))
result = toposort.toposort_flatten(result)
return result
def sort_dependent_tables(table_defs, base_path, illegal_tables, schema,
empty_tables):
deps_dict = {}
for table_def in table_defs:
table_schema = table_def.find('table-schema')
if table_schema.text.lower() != schema:
continue
table_name = table_def.find("table-name")
# print(table_name.text)
disposed = table_def.find("disposed")
if disposed.text != "true":
deps_dict.update({
table_name.text.lower():
get_table_deps(table_name, table_def, deps_dict, empty_tables,
illegal_tables, schema)
})
deps_list = toposort_flatten(deps_dict)
# for dep in deps_list:
# print(dep)
return deps_list
def ordered_task_instances_list(self):
"""
Start from a root task, and read task dependencies recursively.
"""
# NOTE consider Luiti modifies Luigi, same task instances maybe not unique. TODO Fix it
def func(dep_dict, _curr_task):
""" Generate a a dependencies graph. """
required_task_instances = _curr_task.requires()
if not isinstance(required_task_instances, list):
required_task_instances = [required_task_instances]
dep_dict[_curr_task] = filter(
lambda t1: bool(t1) and (not isinstance(t1, RootTask)),
required_task_instances)
if self.check_output_recursive:
dep_dict[_curr_task] = filter(lambda t1: t1.output().exists(),
dep_dict[_curr_task])
dep_dict[_curr_task] = set(dep_dict[_curr_task])
for t1 in dep_dict[_curr_task]:
func(dep_dict, t1)
return dep_dict
# 1. generate a dep dict
task_instances_dep_dict = func(dict(), self.curr_task_intance)
# 2. sort the DAG.
from toposort import toposort_flatten
ordered_task_instances_list = toposort_flatten(task_instances_dep_dict)
return ordered_task_instances_list
def recreateSequence(darcs):
from toposort import toposort_flatten
dependFrom={}
for u,v in darcs:
dependFrom[v]=set([u])
return toposort_flatten(dependFrom)
def resolveDependencies(self):
import toposort
#build dependancy graph
dg = self.dependancyGraph()
#solve the dependency tree
return toposort.toposort_flatten(dg)
def process_spec(specs):
"""Builds mapping class -> set of all descendants."""
tree = dict((name, set(children)) for name, children, _ in specs)
attr_map = dict((name, attrs) for name, _, attrs in specs)
for node in toposort_flatten(tree):
children = tree[node]
for child in set(children):
tree[node] |= tree[child]
return tree, attr_map
def create_base_query(self, sql_generator):
self.anchor_table = self.get_data_table(self.PATIENTS).alias(self.BASE_PREFIX + self.PATIENTS)
anchor_column = self.anchor_table.c.get(QDMConstants.PATIENT_ID_COL).label(self.outer_colname(QDMConstants.PATIENT_ID_COL))
base_from = self.anchor_table
base_select_cols = [anchor_column]
so_list = list(toposort_flatten(self.raw_symbol_table.so_temporal_dependencies))
self.merge_so_lists(so_list, self.data_criteria_names_used)
so_done = []
for var in self.variables.values():
if not var.is_specific_occurrence_target():
sel = var.get_selectable(self, correlate=False)
query = sel.get_selectable()
sel.set_subquery_table(sql_generator.create_table_from_select(var.get_short_name(), query))
query = sel.get_selectable()
for colname in [QDMConstants.UNIQUE_ID, QDMConstants.STARTTIME, QDMConstants.ENDTIME]:
col = sel.get_column(colname)
if col is not None:
base_select_cols.append(col)
base_from = base_from.join(query, sel.get_column(QDMConstants.PATIENT_ID_COL) == anchor_column, isouter=True)
for var in self.specific_occurrence_targets.values():
sel = var.get_selectable(self, correlate=False)
query = sel.get_selectable()
sel.set_subquery_table(sql_generator.create_table_from_select(var.get_short_name(), query))
processed_sos = dict()
for var in self.specific_occurrences.values():
sel = var.get_selectable(self, correlate=False)
root = var.get_raw_id().get_root()
join_conds = [sel.get_column(QDMConstants.PATIENT_ID_COL) == anchor_column]
prior_sos = processed_sos.get(root)
if prior_sos == None:
processed_sos[root] = [sel]
else:
for prior_so in prior_sos:
join_conds.append(sel.get_column(QDMConstants.UNIQUE_ID) !=
prior_so.get_column(QDMConstants.UNIQUE_ID))
prior_sos.append(sel)
base_from = base_from.join(sel.get_from(), and_(*join_conds), isouter=True)
for colname in [QDMConstants.UNIQUE_ID, QDMConstants.STARTTIME, QDMConstants.ENDTIME]:
col = sel.get_column(colname)
if col is not None:
base_select_cols.append(col.label(var.to_so_colname(col.name)))
# base_select = select(base_select_cols).select_from(base_from)
# self.base_select = base_select.alias('patient_base')
# self.anchor_column = self.base_select.corresponding_column(anchor_column)
self.base_select = sql_generator.create_table_from_select('patient_base', select(base_select_cols).select_from(base_from))
for var in self.specific_occurrences.values():
for colname in [QDMConstants.UNIQUE_ID, QDMConstants.STARTTIME, QDMConstants.ENDTIME]:
var.get_selectable(self).set_column(colname, self.base_select.c.get(var.to_so_colname(colname)))
self.anchor_column = self.base_select.c.get(self.outer_colname(QDMConstants.PATIENT_ID_COL))
self.anchor_table = self.base_select
def sortCodes(self):
"""put self.codes in a working evaluation order"""
codeFromOutput = dict((c.outName, c) for c in self.codes)
deps = {}
for c in self.codes:
outName = c.outName
inNames = c.possibleVars.intersection(codeFromOutput.keys())
inNames.discard(outName)
deps[outName] = inNames
self.codes = [codeFromOutput[n] for n in toposort.toposort_flatten(deps)]
def testMiddle(self):
#
graph = {1: set(),
2: set([3]),
3: set()}
expected = deepcopy(graph)
expected[1].add(2)
topo_prioritize(2, graph)
self.assertEquals(graph, expected)
self.assertEqual(toposort_flatten(graph), [3, 2, 1])
def test_sort_flatten(self):
data = {2: {11},
9: {11, 8},
10: {11, 3},
11: {7, 5},
8: {7, 3, 8}, # includes something self-referential
}
expected = [{3, 5, 7}, {8, 11}, {2, 9, 10}]
self.assertEqual(list(toposort(data)), expected)
# now check the sorted results
results = []
for item in expected:
results.extend(sorted(item))
self.assertEqual(toposort_flatten(data), results)
# and the unsorted results. break the results up into groups to compare them
actual = toposort_flatten(data, False)
results = [{i for i in actual[0:3]}, {i for i in actual[3:5]}, {i for i in actual[5:8]}]
self.assertEqual(results, expected)
def toposort_send(signal, sender, **kwargs):
"""Send signal in topological order to all connected receivers."""
from toposort import toposort_flatten
if not signal.receivers:
return []
else:
return [(receiver, receiver(sender, **kwargs))
for receiver in toposort_flatten({
receiver: toposort_extract(receiver)
for receiver in signal.receivers_for(sender)
})]
def __init__(self, thread_id, name, experiment, component_id, max_results, cache_results):
threading.Thread.__init__(self)
self.threadID = thread_id
self.name = name
self.experiment = experiment
self.comp_id = component_id
self.result = {}
self.max_results = max_results
self.cache_results = cache_results
print "Submitting topology to storm. End component", self.comp_id
exp = Experiment.objects.get(pk=self.experiment)
graph = exp.workflow.graph_data
graph_data = {}
print graph
tmp = graph.split(',')
for elem in tmp:
first_node = elem.split(":")[0]
second_node = elem.split(":")[1]
if second_node in graph_data:
depend_nodes = graph_data[second_node]
depend_nodes.add(first_node)
else:
graph_data[second_node] = set()
graph_data[second_node].add(first_node)
topological_graph = toposort_flatten(graph_data)
print "Graph after topological sort", topological_graph
message = {
'exp_id': self.experiment, 'result': self.comp_id,
'graph': topological_graph, 'components': defaultdict()}
for data in topological_graph:
component_id = int(data)
comp = Component.objects.get(pk=component_id)
if comp.operation_type.function_type == 'Create':
if comp.operation_type.function_arg == 'Table':
filename = comp.operation_type.function_subtype_arg
input_data = read_csv(filename)
message['input'] = {}
for elem in list(input_data.columns):
message['input'][elem] = list(input_data[elem])
message['cols'] = list(input_data.columns)
# message['input'] = input_data.to_dict()
serialized_obj = serializers.serialize('json', [comp.operation_type, ])
print "Component_id", component_id, " ", comp.operation_type
message['components'][data] = serialized_obj
print "Message ", message
r = redis.StrictRedis(host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=0)
self.pubsub = r.pubsub(ignore_subscribe_messages=True)
self.pubsub.subscribe("Exp " + str(self.experiment))
ret = r.publish('workflow', json.dumps(message))
print "return", ret
def get_parentmap_toposort(self, reverse=False):
"""
Returns an iterator in topological order over element, parent pairs.
"""
parent_map = self.get_parentmap()
digraph = graph.digraph(parent_map.items())
if reverse:
digraph = graph.reverse(digraph)
for element in toposort_flatten(digraph, sort=False):
yield element, parent_map.get(element, None)
def sort(self, queryset):
"""Sort given the queryset in a topological order.
:param queryset: django queryset
:return: sorted list of `Migration` objects
"""
first_migration = queryset.first()
uids = {migration.uid: migration for migration in (self.filter(
case__ci_project=first_migration.case.ci_project,
) if first_migration else self.all())}
mapping = {
migration.uid: {migration.parent.uid} if migration.parent else set() for migration in queryset}
return [uids[uid] for uid in toposort_flatten(mapping)]
def sort(data):
deps = {}
output = {}
for app in data['assets']:
if app.get('depends'):
deps[app['name']] = set(d['name'] for d in app['depends'])
else:
deps[app['name']] = set()
output[app['name']] = app
#import pprint
#pprint.pprint(toposort_flatten(deps))
#assert False, 'asdf'
return toposort_flatten(deps), output
def _sort_features(features):
graph = {}
for f in features:
graph[f.name] = set(f.deps)
order = toposort_flatten(graph, sort=True)
ret = []
for name in order:
for f in features:
if f.name == name:
ret.append(f)
return ret
def mount_lowerdirs(self):
# First, build a dependency graph in order to avoid duplicate entries
dependencies = {}
pending_deps = set(self.dependencies)
while len(pending_deps) > 0:
name = pending_deps.pop()
if name not in dependencies:
dependencies[name] = set(Container(name).dependencies)
pending_deps |= dependencies[name]
# Then sort it topologically. The list is reversed, because overlayfs
# will check the mounts in order they are given, so the base fs has to
# be the last one.
dependencies = reversed(list(toposort_flatten(dependencies)))
return ':'.join('/var/lib/machines/{}/overlay.fs'.format(dep) for dep in dependencies)
def assignLevel(self):
if not self.changed:
return
dependencyDict = {}
for node in self.nodes:
dependencyDict[node] = node.precursors
self.maxLevel = 0
self.sortedNodes = toposort_flatten(dependencyDict)
for node in self.sortedNodes:
maxLevel = -1
for precursor in node.precursors:
maxLevel = max(maxLevel, node.level)
node.level = maxLevel + 1
self.maxLevel = max(self.maxLevel,node.level)
def get_order(setl_graph):
nodes = collections.defaultdict(set)
for typ in [setl.Extract, setl.Transform, setl.Load]:
for task in setl_graph.subjects(RDF.type, typ):
task = setl_graph.resource(task)
for used in task[prov.used]:
nodes[task.identifier].add(used.identifier)
for usage in task[prov.qualifiedUsage]:
used = usage.value(prov.entity)
nodes[task.identifier].add(used.identifier)
for generated in task.subjects(prov.wasGeneratedBy):
nodes[generated.identifier].add(task.identifier)
return toposort_flatten(nodes)
def ____topologicSortModules(self,path,modules):
dependencyDict = {}
for module in modules:
dependencySet = set()
fModule = open(os.path.join(path,module),'r')
for line in fModule:
line = line.strip()
if line.startswith('include'):
dependency = line.replace('include','').strip()
dependencySet.add(dependency[1:-1])
fModule.close()
dependencyDict[module] = dependencySet
from toposort import toposort_flatten
modules = toposort_flatten(dependencyDict)
return modules
