def test_simple(self):
self.assertEqual(
list(
toposort({
2: {11},
9: {11, 8},
10: {11, 3},
11: {7, 5},
8: {7, 3},
})), [
{3, 5, 7},
{8, 11},
{2, 9, 10},
])
# make sure self dependencies are ignored
self.assertEqual(
list(
toposort({
2: {2, 11},
9: {11, 8},
10: {10, 11, 3},
11: {7, 5},
8: {7, 3},
})), [
{3, 5, 7},
{8, 11},
{2, 9, 10},
])
self.assertEqual(list(toposort({1: set()})), [{1}])
self.assertEqual(list(toposort({1: {1}})), [{1}])
def ordered_steps(self):
dependencies = {
step.name: step.depends() for step in self.steps()
}
# add dependencies on trait-defined steps
for step in self.steps():
dependencies[step.name] |= {
s.name for s in self.steps()
if s.injecting_trait_name() in step.trait_depends()
}
try:
result = list(toposort.toposort(dependencies))
except toposort.CircularDependencyError as de:
dependencies = self._find_and_resolve_publish_trait_circular_dependencies(
dependencies,
cycle_info=de.data,
)
dependencies = self._find_and_resolve_release_trait_circular_dependencies(
dependencies,
cycle_info=de.data,
)
# try again - if there is still a cyclic dependency, this is probably caused
# by a user error - so let it propagate
result = list(toposort.toposort(dependencies))
# result contains a generator yielding tuples of step name in the correct execution order.
# each tuple can/should be parallelised
return result
def test_simple(self):
self.assertEqual(list(toposort({2: {11},
9: {11, 8},
10: {11, 3},
11: {7, 5},
8: {7, 3},
})),
[{3, 5, 7},
{8, 11},
{2, 9, 10},
])
# make sure self dependencies are ignored
self.assertEqual(list(toposort({2: {2, 11},
9: {11, 8},
10: {10, 11, 3},
11: {7, 5},
8: {7, 3},
})),
[{3, 5, 7},
{8, 11},
{2, 9, 10},
])
self.assertEqual(list(toposort({1: set()})),
[{1}])
self.assertEqual(list(toposort({1: {1}})),
[{1}])
def test_cycle(self):
# a simple, 2 element cycle
self.assertRaises(ValueError, list, toposort({1: {2},
2: {1}
}))
# an indirect cycle
self.assertRaises(ValueError, list, toposort({1: {2},
2: {3},
3: {1},
}))
def eval_order_and_gate_lookup(self):
deps = fn.merge(
{a & -2: {b & -2, c & -2}
for a, b, c in self.gates},
{a & -2: set()
for a in self.inputs.values()},
{a & -2: set()
for a, _, _ in self.latches.values()}, # LatchIn
)
latch_deps = {a & -2: {b & -2} for a, b, _ in self.latches.values()}
return list(toposort(deps)), list(toposort(latch_deps))
def test_no_dependencies(self):
self.assertEqual(list(toposort({1: {2},
3: {4},
5: {6},
})),
[{2, 4, 6}, {1, 3, 5}])
self.assertEqual(list(toposort({1: set(),
3: set(),
5: set(),
})),
[{1, 3, 5}])
def test_no_dependencies(self):
self.assertEqual(list(toposort({
1: {2},
3: {4},
5: {6},
})), [{2, 4, 6}, {1, 3, 5}])
self.assertEqual(list(toposort({
1: set(),
3: set(),
5: set(),
})), [{1, 3, 5}])
def get_all_classes_that_needs_schema(class_: type) -> List[type]:
"""
Iterates over the "class_" attributes searching for objects that needs schema (attr classes)
:param type class_:
Target class to return all elements that needs schema.
"""
graph = {}
search_queue = deque()
graph[CaseDescription] = _get_classes(class_)
search_queue += graph[CaseDescription]
while search_queue:
attr_class = search_queue.popleft()
attr_classes = _get_classes(attr_class)
graph[attr_class] = attr_classes
search_queue += graph[attr_class]
# Topological Sorting to keep dependencies correct
from toposort import toposort
result = []
for dependencies in toposort(graph):
# Items are sorted to make the results deterministic
result.extend(sorted(dependencies,
key=operator.attrgetter("__name__")))
return result
def find_min_topology(self): #, ops, topology=[]):
dep_satisfied = lambda dep: (
dep._cacheable and self.using_cache and self.cache is not None and
(dep in self.cache or dep.get_name() in self.cache))
dep_map = dict()
deps_to_resolve = list(self.required_outputs)
while len(deps_to_resolve) > 0:
o = deps_to_resolve[0]
deps_to_resolve = deps_to_resolve[1:]
reqs = self.dep_map[o]
# No matter what, the dag needs to see the requirements
dep_map[o] = reqs
# reqs can be kwargs or args (list)
_iter_reqs = reqs.values() if isinstance(reqs, dict) else reqs
# But only need to resolve those that are not satisfied already
deps_to_resolve += [r for r in _iter_reqs if not dep_satisfied(r)]
dep_sets = {
p: set(d.values()) if isinstance(d, dict) else set(d)
for p, d in dep_map.items()
}
topology = list(toposort(dep_sets))
return topology
def test_dependencies_list(self):
filename = "test_data.json"
target_function = "F"
try:
# Open file with list of dependencies written as a dictionary and multiple correct solutions
with open(filename, "r") as data:
list_of_results = []
dictionary = ast.literal_eval(data.read())
# Using a global variable
TestDependenciesList.filesize = len(dictionary)
# Getting just the first element with the list of dependencies
list_of_dicts = list(toposort(dictionary[0]))
# Create async loop
loop = asyncio.get_event_loop()
list_of_results = loop.run_until_complete(print_dependencies(list_of_dicts, target_function))
# Ignoring first element of the file (Not part of the possible solutions)
for dict in dictionary[1:]:
# Without a subtest, execution would stop after the first failure
with self.subTest(dict=dict):
self.assertEqual(list_of_results, dict)
return (dictionary[1:])
except FileNotFoundError:
print("File not found")
else:
data.close()
def arrangeNodes(dg):
ts = list(toposort.toposort(dg))
xc = 0
ips = {}
for st in ts:
yps = [];
st = list(st)
for si in st:
if si in dg.keys():
ri = list(dg[si])
yp = np.mean([ips[rr][1] for rr in ri])
else:
yp = 0
while yp in yps:
yp += 1
yps.append(yp)
#ysi = np.argsort(yps)
#ys = (np.arange(len(ysi)) - ysi.mean())
ysi = np.arange(len(yps))
ys = yps
for i, yi in zip(ysi, ys):
ips[st[i]] = (xc, yi)
xc += 1
return ips
def load_schema(self):
from metaphor.lrparse.lrparse import parse
self.specs = {}
self.root = Spec('root', self)
schema_data = self._load_schema_data()
self._build_specs(schema_data)
calcs = self._collect_calcs(schema_data)
calc_deps = dict(
(key, set(calc.get('deps', {}))) for key, calc in calcs.items())
sorted_calcs = list(toposort(calc_deps))
for line in sorted_calcs:
for field_str in line:
spec_name, field_name = field_str.split('.')
field_data = schema_data['specs'][spec_name]['fields'][
field_name]
if field_data['type'] == 'calc':
spec = self.specs[spec_name]
# log.debug("Calc create: %s.%s", spec_name, field_name)
self._add_calc(spec, field_name, field_data['calc_str'])
self.calc_trees[(spec.name, field_name)] = parse(
field_data['calc_str'], spec)
for root_name, root_data in schema_data.get('root', {}).items():
self._add_field(self.root,
root_name,
root_data['type'],
target_spec_name=root_data.get('target_spec_name'))
def test_sort_flatten(self):
data = {
2: {11},
9: {11, 8},
10: {11, 3},
11: {7, 5},
8: {7, 3, 8}, # includes something self-referential
}
expected = [{3, 5, 7}, {8, 11}, {2, 9, 10}]
self.assertEqual(list(toposort(data)), expected)
# now check the sorted results
results = []
for item in expected:
results.extend(sorted(item))
self.assertEqual(toposort_flatten(data), results)
# and the unsorted results. break the results up into groups to compare them
actual = toposort_flatten(data, False)
results = [{i
for i in actual[0:3]}, {i
for i in actual[3:5]},
{i
for i in actual[5:8]}]
self.assertEqual(results, expected)
def update_queue(package_queue, to_build_orig, failed, in_progress, built):
if failed:
# If we have failure(s), there is no point in continuing
return
to_build = {k: v for k, v in to_build_orig.items()}
to_remove = set()
for pkg_name, package in in_progress.items():
for child in package.children:
to_remove.add(child)
for rm in to_remove:
to_build.pop(rm, None)
deps_mapping = {
p.name: {d
for d in p.build_time_dependencies() if d not in built}
for p in list(to_build.values())
}
sorted_ordering = [list(deps) for deps in toposort(deps_mapping)]
for item in filter(
lambda i: i in to_build and i not in in_progress and i not in
package_queue.queue and i not in built,
sorted(sorted_ordering[0],
key=lambda k: to_build_orig[k].batch_priority)
if sorted_ordering else []):
package_queue.put(to_build_orig.pop(item))
def topo_sort(nodes):
n = [node.id for node in nodes]
n_dict = dict()
if None in n:
logger.error("node with id None")
for i, node in enumerate(nodes):
dep = []
for arg in node.inputs:
for val in arg:
if val["id"] in n:
dep.append(n.index(val["id"]))
n_dict[i] = set(dep)
sorted_nodes = list(toposort(n_dict))
re = []
for rank in sorted_nodes:
for i in list(rank):
re.append(nodes[i])
return re
def test_toposort():
tf.reset_default_graph()
nodes = util.make_caterpillar_graph(length=2)
graph = linearize_lib.get_graph()
initial = list(toposort(graph))[0]
assert len(initial) == 1
assert list(initial)[0].name == 'merge2'
def test_toposort(): tf.reset_default_graph() nodes = util.make_caterpillar_graph(length=2) graph = linearize_lib.get_graph() initial = list(toposort(graph))[0] assert len(initial) == 1 assert list(initial)[0].name == 'merge2'
def __init__(self, node_type, adj_mat, n_channels):
"""
With a valid and reduced network topology (adj_mat), a cell is constructed
:param node_type: 1d np.array node_type[2xi] in (conv, pool[id]), node_type[2xi + 1] in (1by1, 3by3)
:param adj_mat: 2d np.array upper triangular with zero diagonal
(i,j) entry represents edge from i-node to j-node
:param n_channels: numeric(int)
"""
super(NASBinaryCell, self).__init__()
self.n_nodes = adj_mat.shape[0]
assert len(node_type) == (self.n_nodes - 2) * 2
assert (0 <= node_type).all() and (node_type <= 1).all()
self.adj_mat = adj_mat
topo_input = {v: set(np.where(adj_mat[:, v] == 1)[0]) for v in range(1, self.n_nodes)
if np.sum(adj_mat[:, v]) > 0}
self.topo_order = list(toposort.toposort(topo_input))
assert self.topo_order[0] == {0}
assert self.topo_order[-1] == {self.n_nodes - 1}
self.node0 = None
for i in range(1, self.n_nodes - 1):
n_t, n_s = node_type[2*i-2:2*i]
node = (NodeId() if n_s == 0 else NodeMaxpool3by3()) if n_t == 0 else \
(NodeConv3by3(n_channels) if n_s == 0 else NodeConv5by5(n_channels))
setattr(self, 'node' + str(i), node)
setattr(self, 'node' + str(self.n_nodes - 1), NodeId())
def split_dags(forest):
parents = defaultdict(set)
def find_parents(expr):
for child in expr.children:
parents[child].add(expr)
find_parents(child)
for expr in forest:
find_parents(expr)
def contains(tree, expr):
if tree == expr:
return True
else:
return any(contains(child, expr) for child in tree.children)
exprs = forest + [expr for expr in parents if len(parents[expr]) > 1]
# TODO: This is O(n^3)
deps = {expr: set() for expr in exprs}
for i in xrange(len(exprs)):
for j in xrange(len(exprs)):
if i == j:
continue
if contains(exprs[i], exprs[j]):
deps[exprs[i]].add(exprs[j])
return list(toposort(deps))
def _topologically_sorted(node_dependencies) -> List[Set[Node]]:
"""Topologically group and sort (order) nodes such that no node depends on
a node that appears in the same or a later group.
Raises:
CircularDependencyError: When it is not possible to topologically order
provided nodes.
Returns:
The list of node sets in order of execution. First set is nodes that should
be executed first (no dependencies), second set are nodes that should be
executed on the second step, etc.
"""
def _circle_error_message(error_data: Dict[str, str]) -> str:
"""Error messages provided by the toposort library will
refer to indices that are used as an intermediate step.
This method can be used to replace that message with
one that refers to the nodes' string representations.
"""
circular = [str(node) for node in error_data.keys()]
return "Circular dependencies exist among these items: {}".format(circular)
try:
return list(toposort(node_dependencies))
except ToposortCircleError as error:
message = _circle_error_message(error.data)
raise CircularDependencyError(message) from error
def _construct_toposorted_chain_of_tasks(self) -> Chain:
"""Take the directed graph and toposort so that we can efficiently
organize our workflow, i.e. parallelize where possible.
if we have 2 elements where one of both is an Ellipsis object we need to orchestrate just 1 job.
In the other case we will loop over the toposorted dag and assign a stepfunctions task
or assign multiple tasks in parallel.
Returns: toposorted chain of tasks
"""
self.chain_of_tasks = Chain()
directed_graph_toposorted = list(toposort.toposort(
self.directed_graph))
if self._is_one_task(
directed_graph_toposorted=directed_graph_toposorted):
sfn_task = self.add_task(next(iter(directed_graph_toposorted[0])))
self.chain_of_tasks.append(sfn_task)
else:
for element in directed_graph_toposorted:
if len(element) == 1:
sfn_task = self.add_task(next(iter(element)))
elif len(element) > 1:
sfn_task = self.add_parallel_tasks(element)
else:
raise StepfunctionsWorkflowException(
"cannot have an index in the directed graph with 0 elements"
)
self.chain_of_tasks.append(sfn_task)
return self.chain_of_tasks
def build(self):
"""Build a topological order
"""
topo_sort = list(toposort.toposort(self._build_dependency_dict()))
for i in range(0, len(topo_sort)):
self._topo_sort[i] = topo_sort[i]
# if a bw op has the same order with a fw op,
# then remove the bw op
self._clean_bw_ops()
# if there are non-bw ops in the bw phase,
# then remove them, and do reordering
self._clean_update_ops()
self._reindex()
# build a dict of (op, order)
self._build_order_dict()
# starting order of the backward phase
for i in range(0, len(self._topo_sort)):
ops = self._topo_sort[i]
if (ops & self._grad_ops):
self._bw_starting_order = i
break
def build_dep_graph(input_files, build_info):
pattern = re.compile('[a-zA-Z0-9_-]+')
flatten = lambda l: [item for sublist in l for item in sublist]
dist_to_lines = collections.defaultdict(list)
for i in input_files:
with open(i) as f:
for l in f.readlines():
l = l.strip()
m = pattern.match(l)
if m:
dist_to_lines[m.group()].append(l)
if not build_info:
return [flatten(dist_to_lines.values())]
deps = collections.defaultdict(list)
for dist, info in build_info.items():
for d in info.get("additional_buildtime_deps", []):
deps[dist] += dist_to_lines[d]
graph = {
r: set(deps[n]) if n in deps else set()
for n, rr in dist_to_lines.items() for r in rr
}
result = list(toposort.toposort(graph))
return result
def critical_paths(graph):
def events_earliest_time(vnum, graph, toposeq):
ee = [0] * vnum
for i in toposeq:
for j, w in graph.out_edges(i):
if ee[i] + w > ee[j]:
ee[j] = ee[i] + w
return ee
def event_latest_time(vnum, graph, toposeq, eelast):
le = [eelast] * vnum
for k in range(vnum-2, -1, -1):
i = toposeq[k]
for j, w in graph.out_edges(i):
if le[j] - w < le[i]:
le[i] = le[j] - w
return le
def crt_paths(vnum, graph, ee, le):
crt_actions = []
for i in range(vnum):
for j, w in graph.out_edges(i):
if ee[i] == le[j] - w:
crt_actions.append((i, j, ee[i]))
return crt_actions
toposeq = toposort(graph)
if not toposeq: # 不存在拓扑序列，失败
return False
vnum = graph.vertex_num()
ee = events_earliest_time(vnum, graph, toposeq)
le = event_latest_time(vnum, graph, toposeq, ee[vnum-1])
return crt_paths(vnum, graph, ee, le)
def build_dep_graph(args):
pattern = re.compile('[a-zA-Z0-9_-]+')
requirements = {}
for i in args.input:
with open(i) as f:
for l in f.readlines():
l = l.strip()
m = pattern.match(l)
if m:
requirements[m.group()] = l
if not args.build_dep:
return [requirements.values()]
deps = {}
for i in args.build_dep:
k, v = i.split('=')
if k not in requirements:
continue
if k not in deps:
deps[k] = []
deps[k].append(requirements[v])
graph = {
r: set(deps[n]) if n in deps else set()
for n, r in requirements.items()
}
result = list(toposort.toposort(graph))
return result
def run(self):
if not self.run_order:
self.run_order = list(chain.from_iterable(toposort(self.adjList)))
for n in self.run_order: # Each node to be processed
n.next_frame()
n.run()
def _sort_tasks(task_list):
"""
Take the list of tasks in any order and sort them topologically
so the resulting list and be executed correctly.
:param task_list: List of LocalTask objects
:return: Sorted list of LocalTask objects
"""
# Create a list of
tasks_w_deps = {}
for task in task_list:
deps = []
for port_name in task.inputs.portnames:
port = task.inputs.__getattribute__(port_name)
if isinstance(port.value, local_task.Directory):
if port.value.parent not in task_list:
raise LocalWorkflowError(
'Task %s is missing from workflow' %
port.value.parent.type)
deps.append(port.value.parent)
tasks_w_deps[task] = set(deps)
sorted_list = []
for x in toposort(tasks_w_deps):
sorted_list += list(x)
return sorted_list
def __str__(self):
if not len(self._memories_dictionary):
return "<empty graph>"
# Sort memories topologically
def get_dependencies(n):
return set(n.precursor_keys)
to_be_topologically_sorted = {memory.key: get_dependencies(memory) for memory in self._memories_dictionary.values()}
topologically_sorted = [l for g in toposort(to_be_topologically_sorted) for l in g]
# Find longest strings so we can pad our strings to equal length later
def get_precursors_str(n):
return ', '.join(self._memories_dictionary[n].precursor_keys)
data_to_print = [(get_precursors_str(n), n, self._memories_dictionary[n].is_stale) for n in topologically_sorted]
longest_precursors_str = max([len(d[0]) for d in data_to_print])
longest_memory_key = max([len(d[1]) for d in data_to_print])
# Build output str in format: "[precursors] --> memory" along with an indicate of stale-ness
result_str = ""
for data in data_to_print:
left_side = data[0].rjust(longest_precursors_str)
right_side = data[1].ljust(longest_memory_key)
stale_indicator = " (stale)" if data[2] else ""
result_str += f"{left_side} --> {right_side}{stale_indicator}\n"
result_str = result_str.rstrip('\n')
return result_str
def build_dep_graph(args):
pattern = re.compile('[a-zA-Z0-9_-]+')
flatten = lambda l: [item for sublist in l for item in sublist]
dist_to_lines = collections.defaultdict(list)
for i in args.input:
with open(i) as f:
for l in f.readlines():
l = l.strip()
m = pattern.match(l)
if m:
dist_to_lines[m.group()].append(l)
if not args.build_dep:
return [flatten(dist_to_lines.values())]
deps = collections.defaultdict(list)
for i in args.build_dep:
k, v = i.split('=')
if k not in dist_to_lines:
continue
deps[k] += dist_to_lines[v]
graph = {
r: set(deps[n]) if n in deps else set()
for n, rr in dist_to_lines.items() for r in rr
}
result = list(toposort.toposort(graph))
return result
def remove_correlated_features(df, r=0.99):
names_correlated = find_correlated_features(df, r=r)
sorted_names = list(toposort(names_correlated))
removed_features = list()
for names in sorted_names:
for name in names:
# If feature not on the left - remove it
if name not in names_correlated.keys():
for key, value in names_correlated.items():
try:
names_correlated[key].remove(name)
removed_features.append(name)
except KeyError:
pass
for name in names_correlated.keys():
if not names_correlated[name]:
for key, value in names_correlated.items():
try:
names_correlated[key].remove(name)
removed_features.append(name)
names_correlated.pop(name)
except KeyError:
pass
names = list(set(removed_features))
print("Removed features : ")
print(names)
return df.drop(names, axis=1)
def toposort_symbols(symbols: SymbolDef,
field: Optional[str] = None) -> SymbolDef:
"""
Topologically sort symbol definitions according to their interdependency
:param symbols:
symbol definitions
:param field:
field of definition.values() that is used to compute interdependency
:return:
ordered symbol definitions
"""
sorted_symbols = toposort({
identifier: {
s for s in (
definition[field] if field is not None else definition
).free_symbols
if s in symbols
}
for identifier, definition
in symbols.items()
})
return {
s: symbols[s]
for symbol_group in sorted_symbols
for s in symbol_group
}
def arrangeNodes(dg):
ts = list(toposort.toposort(dg))
xc = 0
ips = {}
for st in ts:
yps = []
st = list(st)
for si in st:
if si in dg.keys():
ri = list(dg[si])
yp = np.mean([ips[rr][1] for rr in ri])
else:
yp = 0
while yp in yps:
yp += 1
yps.append(yp)
#ysi = np.argsort(yps)
#ys = (np.arange(len(ysi)) - ysi.mean())
ysi = np.arange(len(yps))
ys = yps
for i, yi in zip(ysi, ys):
ips[st[i]] = (xc, yi)
xc += 1
return ips
def _sort_tasks(task_list):
"""
Take the list of tasks in any order and sort them topologically
so the resulting list and be executed correctly.
:param task_list: List of LocalTask objects
:return: Sorted list of LocalTask objects
"""
# Create a list of
tasks_w_deps = {}
for task in task_list:
deps = []
for port_name in task.inputs.portnames:
port = task.inputs.__getattribute__(port_name)
if isinstance(port.value, local_task.Directory):
if port.value.parent not in task_list:
raise LocalWorkflowError('Task %s is missing from workflow' % port.value.parent.type)
deps.append(port.value.parent)
tasks_w_deps[task] = set(deps)
sorted_list =[]
for x in toposort(tasks_w_deps):
sorted_list += list(x)
return sorted_list
def get_topology(self) -> List[Set[EntityType]]:
map_of_dependencies = {}
for entity_type in self.included_entity_types:
map_of_dependencies[entity_type] = []
for target in self.get_dependencies(entity_type, False):
map_of_dependencies[entity_type].append(target)
return list(toposort(map_of_dependencies))
def main():
args = parse_args()
logging.basicConfig(level=args.loglevel)
if has_path is False or has_toposort is False:
LOG.error('Your python environment is missing required modules')
sys.exit(1)
dirs = {}
for item in (Path(dir) for dir in args.dirs):
if item.name.startswith('.'):
continue
if not item.is_dir():
LOG.warn('skipping %s (not a directory)', item)
continue
dirs[item] = {
'requires': [],
'required_by': [],
'provides': [],
}
this = dirs[item]
for reqtype in this.keys():
LOG.debug('check %s %s', item.name, reqtype)
try:
with (item / '.deps' / reqtype).open() as fd:
LOG.debug('reading %s deps for %s', reqtype, item.name)
this[reqtype] = fd.read().splitlines()
except IOError:
continue
# generated provided_by dictionary
provided_by = {}
for k, v in dirs.items():
provided_by[k.name] = k
for provide in v['provides']:
provided_by[provide] = k
order = {}
for k, v in dirs.items():
order.setdefault(k, set())
for req in v['requires']:
LOG.debug('%s requires %s (provided by %s)',
k, req, provided_by[req])
order[k].add(provided_by[req])
for req in v['required_by']:
LOG.debug('%s is required by %s (via %s)',
k, provided_by[req], req)
order.setdefault(provided_by[req], set()).add(k)
order = [item for stage in toposort.toposort(order)
for item in stage]
print ' '.join(str(x) for x in order)
def get_relation_view():
_views = PreferenceRelationView.get_by(to_dict=True)
views = []
if current_app.config.get("USE_ACL"):
for i in _views:
try:
if ACLManager().has_permission(i.get('name'),
ResourceTypeEnum.RELATION_VIEW,
PermEnum.READ):
views.append(i)
except AbortException:
pass
else:
views = _views
view2cr_ids = dict()
result = dict()
name2id = list()
for view in views:
view2cr_ids.setdefault(view['name'], []).extend(json.loads(view['cr_ids']))
name2id.append([view['name'], view['id']])
id2type = dict()
for view_name in view2cr_ids:
for i in view2cr_ids[view_name]:
id2type[i['parent_id']] = None
id2type[i['child_id']] = None
topo = {i['child_id']: {i['parent_id']} for i in view2cr_ids[view_name]}
leaf = list(set(toposort.toposort_flatten(topo)) - set([j for i in topo.values() for j in i]))
leaf2show_types = {i: [t['child_id'] for t in CITypeRelation.get_by(parent_id=i)] for i in leaf}
node2show_types = copy.deepcopy(leaf2show_types)
def _find_parent(_node_id):
parents = topo.get(_node_id, {})
for parent in parents:
node2show_types.setdefault(parent, []).extend(node2show_types.get(_node_id, []))
_find_parent(parent)
if not parents:
return
for l in leaf:
_find_parent(l)
for node_id in node2show_types:
node2show_types[node_id] = [CITypeCache.get(i).to_dict() for i in set(node2show_types[node_id])]
result[view_name] = dict(topo=list(map(list, toposort.toposort(topo))),
topo_flatten=list(toposort.toposort_flatten(topo)),
leaf=leaf,
leaf2show_types=leaf2show_types,
node2show_types=node2show_types,
show_types=[CITypeCache.get(j).to_dict()
for i in leaf2show_types.values() for j in i])
for type_id in id2type:
id2type[type_id] = CITypeCache.get(type_id).to_dict()
return result, id2type, sorted(name2id, key=lambda x: x[1])
def load(shuffle=False):
# contains names like 'fullscreen', not 'porcupine.plugins.fullscreen'
plugin_names = {
name
for finder, name, is_pkg in pkgutil.iter_modules(plugin_paths)
if not name.startswith('_')
}
log.info("found %d plugins", len(plugin_names))
plugin_infos = {} # {name: (setup_before, setup_after, setup_func)}
for name in plugin_names.copy():
log.debug("importing %s", name)
try:
module = importlib.import_module('rbtk.plugins.' + name)
setup_before = set(getattr(module, 'setup_before', []))
setup_after = set(getattr(module, 'setup_after', []))
setup = module.setup
except Exception:
log.exception("problem with importing %s", name)
continue
# now we know that the plugin is ok, we can add its stuff to
# dependencies and setup_funcs
plugin_infos[name] = (setup_before, setup_after, setup)
# setup_before and setup_after may contain names of plugins that are
# not installed because they are for controlling the loading order,
# not for requiring dependencies
def valid_name(name):
return name in plugin_infos
dependencies = {name: set() for name in plugin_infos}
for name, (setup_before, setup_after, setup) in plugin_infos.items():
dependencies[name].update(filter(valid_name, setup_after))
for reverse_dependency in filter(valid_name, setup_before):
dependencies[reverse_dependency].add(name)
# the toposort will partially work even if there's a circular
# dependency, the CircularDependencyError is raised after doing
# everything possible (see source code)
loading_order = []
try:
for names in map(list, toposort.toposort(dependencies)):
if shuffle:
random.shuffle(names)
else:
names.sort()
loading_order.extend(names)
except toposort.CircularDependencyError as e:
parts = map("{} depends on {}".format, e.data.items())
log.error("circular dependency: %s", ', '.join(parts))
for name in loading_order:
*junk, setup = plugin_infos[name]
log.debug("running %s.setup()", name)
try:
setup()
except Exception:
log.exception("%s.setup() doesn't work", name)
def run_setup_functions(shuffle: bool) -> None:
imported_infos = _get_successfully_imported_infos()
# setup_before and setup_after may contain names of plugins that are not
# installed because they are for controlling the loading order. That's fine
# because toposort ignores missing things.
dependencies: Dict[str, Set[str]] = {}
for info in imported_infos:
dependencies.setdefault(info.name, set()).update(info.setup_after)
for reverse_dependency in info.setup_before:
dependencies.setdefault(reverse_dependency, set()).add(info.name)
# the toposort will partially work even if there's a circular
# dependency, the CircularDependencyError is raised after doing
# everything possible (see source code)
loading_order: List[str] = []
try:
# https://github.com/python/mypy/issues/9253
make_list: Callable[[Iterable[str]], List[str]] = list
for names in map(make_list, toposort.toposort(dependencies)):
if shuffle:
# for plugin developers wanting to make sure that the
# dependencies specified in setup_before and setup_after
# are correct
random.shuffle(names)
else:
# for consistency in UI (e.g. always same order of menu items)
names.sort()
loading_order.extend(names)
loadable_infos = imported_infos
except toposort.CircularDependencyError as e:
log.exception("circular dependency")
for info in imported_infos:
if info.name in loading_order:
loadable_infos.append(info)
else:
info.status = Status.CIRCULAR_DEPENDENCY_ERROR
parts = ', '.join(f"{a} depends on {b}"
for a, b in e.data.items())
info.error = f"Circular dependency error: {parts}"
loadable_infos.sort(key=(lambda info: loading_order.index(info.name)))
for info in loadable_infos:
start = time.time()
try:
assert info.module is not None
info.module.setup()
except Exception:
log.exception(f"{info.name}.setup() doesn't work")
info.status = Status.SETUP_FAILED
info.error = traceback.format_exc()
else:
info.status = Status.ACTIVE
duration = time.time() - start
log.debug("ran %s.setup() in %.3f milliseconds", info.name,
duration * 1000)
def test_input_not_modified_when_cycle_error(self):
data = {1: {2},
2: {1},
3: {4},
}
orig = data.copy()
self.assertRaises(ValueError, list, toposort(data))
self.assertEqual(data, orig)
def number_of_strongly_connected_components(adj, rev_adj):
visited = set()
cc = 0
order = toposort(rev_adj)
for v in order:
if v not in visited:
explore(v, adj, visited, cc)
cc += 1
return cc
def calculate_dependencies():
"""Calculate test dependencies
First do a topological sorting based on the dependencies.
Then sort the different dependency groups based on priorities.
"""
order = []
for g in toposort(merge_dicts(dependencies, soft_dependencies)):
for t in sorted(g, key=lambda x: (priorities[x], x)):
order.append(t)
return order
def test_input_not_modified(self):
data = {2: {11},
9: {11, 8},
10: {11, 3},
11: {7, 5},
8: {7, 3, 8}, # includes something self-referential
}
orig = data.copy()
results = list(toposort(data))
self.assertEqual(data, orig)
def ops_in_toposorted_order(ops):
"""Produces ops in deterministic order such that children are executed
after parents"""
graph_dict = tf_ops_to_graph(ops)
toposorted = toposort.toposort(graph_dict)
ops = []
# toposort assumes children are dependencies, reverse order
for op_set in reversed(list(toposorted)):
ops.extend(sorted(op_set, key=lambda op: op.name))
return ops
def test_strings(self):
self.assertEqual(list(toposort({'2': {'11'},
'9': {'11', '8'},
'10': {'11', '3'},
'11': {'7', '5'},
'8': {'7', '3'},
})),
[{'3', '5', '7'},
{'8', '11'},
{'2', '9', '10'},
])
def calculate_dependencies():
"""Calculate test dependencies
First do a topological sorting based on the dependencies.
Then sort the different dependency groups based on priorities.
"""
order = []
for group in toposort(dependencies):
priority_sorted_group = sorted(group, key=lambda x: (priorities[x], x))
order.extend(priority_sorted_group)
return order
def main():
CARS = 10
START_PTS = []
END_PTS = []
for i in xrange(CARS):
START_PTS += [(100+i*30, 500, -math.pi/2)]
START_PTS += [(100+i*30, 400, -math.pi/2)]
END_PTS += [(370-i*30, 100, -math.pi/2)]
END_PTS += [(370-i*30, 200, -math.pi/2)]
dist_matrix = []
for start_pt in START_PTS:
dist_vec = []
for end_pt in END_PTS:
dist_vec.append(dubins.path_length(start_pt, end_pt, settings.TURNING_RADIUS))
print dist_vec
dist_matrix.append(dist_vec)
m = munkres.Munkres()
indices = m.compute(dist_matrix)
d_pt_match = {}
for start_pt_ind, end_pt_ind in indices:
d_pt_match[start_pt_ind] = end_pt_ind
cars = []
for i in xrange(CARS):
cars.append((START_PTS[i], END_PTS[d_pt_match[i]]))
print cars[-1]
COLLISION_DIST = 5
d_collisions = {}
for i, (start_pt, end_pt) in enumerate(cars):
d_collisions[i] = set()
path, t = dubins.path_sample(start_pt, end_pt, settings.TURNING_RADIUS, 0.1)
for j, (temp_start_pt, temp_end_pt) in enumerate(cars):
if j == i:
continue
for pt in path:
if norm(sub_vecs(pt, temp_end_pt)) < COLLISION_DIST:
d_collisions[i].add(j)
print d_collisions
cars_order = list(toposort.toposort(d_collisions))
for car_set in cars_order[::-1]:
print "Creating cars", car_set
for car_ind in car_set:
thread.start_new_thread(main_loop, ((cars[car_ind][0][0], cars[car_ind][0][1]), cars[car_ind][0][2],
(cars[car_ind][1][0], cars[car_ind][1][1]), cars[car_ind][1][2]))
time.sleep(10)
def ranks2order(ranks, majority = False):
big = pd.DataFrame([[(ranks[j] > ranks[i]).sum()
for i in ranks.columns]
for j in ranks.columns])
big.columns = ranks.columns
big.index = ranks.columns
if not majority:
majority = big.max().max()
graph = sorted(big[big >= majority].stack().index.tolist())
big_dict = dict([(x, set(a[1] for a in y)) for (x, y)
in itertools.groupby(graph, key=lambda x: x[0])])
true_order = list(toposort.toposort(big_dict))
print("Majority:", majority / len(ranks))
return true_order
def find_sync_groups(containers):
deps = dict()
name2container = dict()
for container in containers:
name2container[container.name] = container
deps[container.name] = container.dependency_names()
synch_groups = []
for synch_group_names in toposort.toposort(deps):
synch_group = set()
for container_name in synch_group_names:
synch_group.add(name2container[container_name])
synch_groups.append(synch_group)
return synch_groups
def build_groups(self):
"""Return topological sort of dependency groups, i.e,
list of sets of containers which can be built in groups."""
result = []
for t in list(toposort.toposort(self.build_depends())):
if t == set(['']): # We don't have to build the null dependency
continue
group = []
for tag in t:
container_name = self.container_name_from_tag(tag)
container = self.get_container(container_name)
if container.buildable:
group.append(container_name)
result.append(group)
return result
def _sort_modules(self):
modules = {}
for instance in self._instances:
if not self._instances[instance]._disabled:
_deps = self._instances[instance]._depends
_afters = self._instances[instance]._after
modules[instance] = _deps.union(_afters)
try:
return [m for m in toposort(modules) if m in self._instances]
except ValueError as e:
circular = findall('.*?\(\'?(\S+?)\'?,', e.args[0])
for instance in circular:
self._instances[instance]._disabled = True
debug(_('Circular dependencies found, disabled modules: %s')
% ', '.join(circular))
return self._sort_modules()
def tf_toposort(ts, within_ops=None):
all_ops = ge.get_forward_walk_ops([x.op for x in ts], within_ops=within_ops)
deps = {}
for op in all_ops:
for o in op.outputs:
deps[o] = set(op.inputs)
sorted_ts = toposort(deps)
# only keep the tensors from our original list
ts_sorted_lists = []
for l in sorted_ts:
keep = list(set(l).intersection(ts))
if keep:
ts_sorted_lists.append(keep)
return ts_sorted_lists
def tf_toposort(ts_inp, within_ops=None):
""" Tensorflow topological sort """
all_ops = ge.get_forward_walk_ops([x.op for x in ts_inp], within_ops=within_ops)
deps = {}
for tf_op in all_ops:
for outp in tf_op.outputs:
deps[outp] = set(tf_op.inputs)
sorted_ts = toposort(deps)
# only keep the tensors from our original list
ts_sorted_lists = []
for lst in sorted_ts:
keep = list(set(lst).intersection(ts_inp))
if keep:
ts_sorted_lists.append(keep)
return ts_sorted_lists
def answer():
f = open('p079_keylog.txt', 'r')
data = list(set(map(int, f.read().split())))
graph = {}
for datum in data:
datum = str(datum)
if datum[2] in graph:
graph[datum[2]] |= {datum[0], datum[1]}
else:
graph[datum[2]] = {datum[0], datum[1]}
if datum[1] in graph:
graph[datum[1]] |= {datum[0]}
else:
graph[datum[1]] = {datum[0]}
graph = list(map(list, list(toposort(graph))))
graph = [item[0] for item in graph]
return ''.join(graph)
def sort_migrations(migration_classes, flatten=False):
migration_dependency_graph = {
m.migration_id: set() if m.dependencies is None else set(m.dependencies)
for m in migration_classes
}
migration_classes_by_id = {
m.migration_id: m for m in migration_classes
}
migration_id_order = toposort(migration_dependency_graph)
migration_execution_sets = [
{migration_classes_by_id[migration_id] for migration_id in execution_set}
for execution_set in migration_id_order
]
if flatten:
return list(itertools.chain.from_iterable(migration_execution_sets))
else:
return migration_execution_sets
def toposort_recipes(recipes):
metadata = list(get_metadata(recipes))
name2recipe = defaultdict(list)
for meta, recipe in zip(metadata, recipes):
name2recipe[meta.get_value("package/name")].append(recipe)
def get_inner_deps(dependencies):
for dep in dependencies:
name = dep.split()[0]
if name in name2recipe:
yield name
dag = {
meta.get_value("package/name"): set(get_inner_deps(chain(get_deps(meta), get_deps(meta, build=False))))
for meta in metadata
}
# note difference here from build-packages.py
return toposort(dag)
def test_objects(self):
o2 = object()
o3 = object()
o5 = object()
o7 = object()
o8 = object()
o9 = object()
o10 = object()
o11 = object()
self.assertEqual(list(toposort({o2: {o11},
o9: {o11, o8},
o10: {o11, o3},
o11: {o7, o5},
o8: {o7, o3, o8},
})),
[{o3, o5, o7},
{o8, o11},
{o2, o9, o10},
])
def test_sort_flatten(self):
data = {2: {11},
9: {11, 8},
10: {11, 3},
11: {7, 5},
8: {7, 3, 8}, # includes something self-referential
}
expected = [{3, 5, 7}, {8, 11}, {2, 9, 10}]
self.assertEqual(list(toposort(data)), expected)
# now check the sorted results
results = []
for item in expected:
results.extend(sorted(item))
self.assertEqual(toposort_flatten(data), results)
# and the unsorted results. break the results up into groups to compare them
actual = toposort_flatten(data, False)
results = [{i for i in actual[0:3]}, {i for i in actual[3:5]}, {i for i in actual[5:8]}]
self.assertEqual(results, expected)
def resolve_dep_order(images_config):
# Figure out what images are provided by this config
# Anything not provided is assumed to exist already
provided_images = dict()
for image_name,image_config in images_config.items():
if container_system_option_names.isdisjoint(image_config.keys()):
provided_images[image_name] = image_name
else:
for commit_spec in optional_plural(image_config, "commits"):
dest = parse_dest_spec(commit_spec, "commits", image_name)
provided_images[dest.image] = image_name
for persist_spec in optional_plural(image_config, "persists"):
dest = parse_dest_spec(commit_spec, "persists", image_name)
provided_images[dest.image] = image_name
# Figure out the images required (among those provided) by images in this config
deps = dict()
for image_name,image_config in images_config.items():
req = set()
deps[image_name] = req
if container_system_option_names.isdisjoint(image_config.keys()):
from_name = image_config["from"].split(":", 1)[0]
if from_name in provided_images:
req.add(from_name)
else:
sys_config = nagoya.cli.cfg.read_one(image_config["system"])
for cont_config in sys_config.values():
image_name = cont_config["image"].split(":", 1)[0]
if image_name in provided_images:
req.add(image_name)
# Toposort to sync groups, use original order of keys to order within groups
image_names = []
for group in toposort.toposort(deps):
image_names.extend(sorted(group, key=lambda n: images_config.keys().index(n)))
return image_names
def run_annotators(self, document_chunk):
for group in toposort(self.dependency_graph):
current_results = []
for annotation_tag in group:
annotator = self.annotators[annotation_tag]
#task_d = [{dep: doc.annotations[dep] for dep in annotator.dependsOn()} for doc in document_chunk]
task_data = document_chunk
async_res = annotator.get_annotations_delay(task_data)
current_results.append((annotation_tag, async_res))
# todo: we can save time by not waiting for all strategies, but only ones that are dependencies
for annotation_tag, async_result in current_results:
document_annotations = async_result.get()
assert len(document_chunk) == len(document_annotations), "Annotator %s did not return the correct number of annotations!"
if async_result.status != 'SUCCESS':
ln.error("Annotator %s threw an error!", annotation_tag)
if isinstance(async_result.result, Exception):
raise async_result.result
for annotation, document in zip(document_annotations, document_chunk):
document.annotations[annotation_tag] = annotation
return document_chunk
def test_toposort(): nodes = make_caterpillar_graph(length=2) graph = linearize.get_graph() print(list(toposort.toposort(graph)))
