def add_edges_from(self, ebunch, attr_dict=None, **attr):
"""Add all the edges in ebunch.
Parameters
----------
ebunch : container of edges
Each edge given in the container will be added to the
graph. The edges must be given as as 2-tuples (u,v) or
3-tuples (u,v,d) where d is a dictionary containing edge
data.
attr_dict : dictionary, optional (default= no attributes)
Dictionary of edge attributes. Key/value pairs will
update existing data associated with each edge.
attr : keyword arguments, optional
Edge data (or labels or objects) can be assigned using
keyword arguments.
Enusres No cycles are in this directed graph (DAG)
"""
# set up attribute dict
if attr_dict is None:
attr_dict = attr
else:
try:
attr_dict.update(attr)
except AttributeError:
raise NetworkXError("The attr_dict argument must be a dict.")
# process ebunch
for e in ebunch:
ne = len(e)
if ne == 3:
u, v, dd = e
assert hasattr(dd, "update")
elif ne == 2:
u, v = e
dd = {}
else:
raise NetworkXError("Edge tuple %s must be a 2-tuple or 3-tuple." % (e,))
if u not in self.succ:
self.succ[u] = {}
self.pred[u] = {}
self.node[u] = {}
if v not in self.succ:
self.succ[v] = {}
self.pred[v] = {}
self.node[v] = {}
datadict = self.adj[u].get(v, {})
datadict.update(attr_dict)
datadict.update(dd)
self.succ[u][v] = datadict
self.pred[v][u] = datadict
# make sure it's a DAG
try:
networkx.topological_sort(self)
except networkx.NetworkXUnfeasible:
self.remove_edge(u, v)
raise networkx.NetworkXUnfeasible(
"Graph contains a cycle! DAG is acyclic! Edge " + u + "-" + v + " cannot be added."
)
def evaluate(self, x):
# XXX: this is a massive hack
x = x[::-1]
assert self.circuit
g = self.circuit.copy()
for node in nx.topological_sort(g):
if "gate" not in g.node[node]:
if g.node[node]["label"][0].startswith("x"):
g.add_node(node, value=int(x[node]))
else:
g.add_node(node, value=int(g.node[node]["value"]))
elif g.node[node]["gate"] in ("ADD", "MUL"):
keys = g.pred[node].keys()
if len(keys) == 1:
idx1 = idx2 = keys[0]
else:
assert len(keys) == 2
idx1 = g.pred[node].keys()[0]
idx2 = g.pred[node].keys()[1]
if g.node[node]["gate"] == "ADD":
value = g.node[idx1]["value"] + g.node[idx2]["value"]
elif g.node[node]["gate"] == "MUL":
value = g.node[idx1]["value"] * g.node[idx2]["value"]
g.add_node(node, value=value)
else:
raise Exception("Unable to evaluate")
idx = nx.topological_sort(g)[-1]
return g.node[idx]["value"] != 0
def evaluate(self, x):
# XXX: this is a massive hack
x = x[::-1]
assert self.circuit
g = self.circuit.copy()
for node in nx.topological_sort(g):
if 'gate' not in g.node[node]:
if g.node[node]['label'][0].startswith('x'):
g.add_node(node, value=int(x[node]))
else:
g.add_node(node, value=int(g.node[node]['value']))
elif g.node[node]['gate'] in ('ADD', 'MUL', 'SUB'):
keys = g.pred[node].keys()
if len(keys) == 1:
idx1 = idx2 = keys[0]
else:
assert(len(keys) == 2)
idx1 = g.pred[node].keys()[0]
idx2 = g.pred[node].keys()[1]
if g.node[node]['gate'] == 'ADD':
value = g.node[idx1]['value'] + g.node[idx2]['value']
elif g.node[node]['gate'] == 'SUB':
value = g.node[idx1]['value'] - g.node[idx2]['value']
elif g.node[node]['gate'] == 'MUL':
value = g.node[idx1]['value'] * g.node[idx2]['value']
g.add_node(node, value=value)
else:
raise Exception('Unable to evaluate')
idx = nx.topological_sort(g)[-1]
return g.node[idx]['value'] != 0
def ensure_dependencies(request):
r"""
CommandLine:
python -m dtool.base --exec-BaseRequest.ensure_dependencies
Example:
>>> # ENABLE_DOCTEST
>>> from dtool.base import * # NOQA
>>> from dtool.example_depcache import testdata_depc
>>> depc = testdata_depc()
>>> request = depc.new_request('vsmany', [1, 2], [2, 3, 4])
>>> request.ensure_dependencies()
"""
import networkx as nx
depc = request.depc
if False:
dependencies = nx.ancestors(depc.graph, request.tablename)
subgraph = depc.graph.subgraph(set.union(dependencies, {request.tablename}))
dependency_order = nx.topological_sort(subgraph)
root = dependency_order[0]
[nx.algorithms.dijkstra_path(subgraph, root, start)[:-1] +
nx.algorithms.dijkstra_path(subgraph, start, request.tablename)
for start in dependency_order]
graph = depc.graph
root = list(nx.topological_sort(graph))[0]
edges = graph.edges()
#parent_to_children = ut.edges_to_adjacency_list(edges)
child_to_parents = ut.edges_to_adjacency_list([t[::-1] for t in edges])
to_root = {request.tablename:
ut.paths_to_root(request.tablename, root, child_to_parents)}
from_root = ut.reverse_path(to_root, root, child_to_parents)
dependency_levels_ = ut.get_levels(from_root)
dependency_levels = ut.longest_levels(dependency_levels_)
true_order = ut.flatten(dependency_levels)[1:-1]
#print('[req] Ensuring %s request dependencies: %r' % (request, true_order,))
ut.colorprint(
'[req] Ensuring request %s dependencies: %r' % (request, true_order,), 'yellow')
for tablename in true_order:
table = depc[tablename]
if table.ismulti:
pass
else:
# HACK FOR IBEIS
all_aids = ut.flat_unique(request.qaids, request.daids)
depc.get_rowids(tablename, all_aids)
pass
pass
#zip(depc.get_implicit_edges())
#zip(depc.get_implicit_edges())
#raise NotImplementedError('todo')
#depc = request.depc
#parent_rowids = request.parent_rowids
#config = request.config
#rowid_dict = depc.get_all_descendant_rowids(
# request.tablename, root_rowids, config=config)
pass
def attempt_edge(graph, a, b):
a_gate, a_port = a
b_gate, b_port = b
graph.add_edge(a_gate, b_gate)
try:
nx.topological_sort(graph)
except nx.NetworkXUnfeasible:
graph.remove_edge(a_gate, b_gate)
broken_up_links.append((a, b))
def test_nbunch_argument(self):
G=nx.DiGraph()
G.add_edges_from([(1,2), (2,3), (1,4), (1,5), (2,6)])
assert_equal(nx.topological_sort(G), [1, 2, 3, 6, 4, 5])
assert_equal(nx.topological_sort_recursive(G), [1, 5, 4, 2, 6, 3])
assert_equal(nx.topological_sort(G,[1]), [1, 2, 3, 6, 4, 5])
assert_equal(nx.topological_sort_recursive(G,[1]), [1, 5, 4, 2, 6, 3])
assert_equal(nx.topological_sort(G,[5]), [5])
assert_equal(nx.topological_sort_recursive(G,[5]), [5])
def _compute_lattice_monotonicity(self, reds, pis):
"""
Infer the redundancy and partial information of lattice elements via lattice monotonicity.
Parameters
----------
reds : dict
Currently known redundancy values.
pis : dict
Currently known partial information values.
Returns
-------
reds : dict
Updated redundancy values.
pis : dict
Updated partial information values.
"""
# everything below a redundancy of 0 is a redundancy of 0
nodes = list(nx.topological_sort(self._lattice))
while nodes:
node = nodes.pop(0)
if node in reds and np.isclose(0, reds[node]):
for n in descendants(self._lattice, node):
if n not in reds:
reds[n] = 0
nodes.remove(n)
# everything above a redundancy of I(inputs, output) is I(inputs, output)
nodes = list(reversed(list(nx.topological_sort(self._lattice))))
while nodes:
node = nodes.pop(0)
if node in reds and np.isclose(reds[node], self._total):
for n in ascendants(self._lattice, node):
if n not in reds:
reds[n] = self._total
nodes.remove(n)
# if redundancy of A == redundancy of B, then for all A -> C -> B, redundancy of C = redundancy of A, B
tops = [node for node in self._lattice if node in reds and any((n not in reds) for n in self._lattice[node])]
bottoms = [node for node in self._lattice if
node in reds and any((n not in reds) for n in self._lattice.reverse()[node])]
for top, bottom in product(tops, bottoms):
if np.isclose(reds[top], reds[bottom], atol=1e-5, rtol=1e-5):
for path in nx.all_simple_paths(self._lattice, top, bottom):
for node in path[1:-1]:
if node not in reds:
reds[node] = reds[top]
# if redundancy of A is equal to the redundancy any of A's children, then pi(A) = 0
for node in self._lattice:
if node not in pis:
if node in reds and all(n in reds for n in self._lattice[node]) and self._lattice[node]:
if any(np.isclose(reds[n], reds[node], atol=1e-5, rtol=1e-5) for n in self._lattice[node]):
pis[node] = 0
return reds, pis
def main():
log = logging.getLogger(__name__)
logging.basicConfig(level = logging.INFO)
backend = foradage.RECAST_Backend(2)
g = adage.mk_dag()
global_context = {
'workdir':'/Users/lukas/Code/code-snippets/cap-schema-drafts/steer',
'dataset':'user15.lheinric.p123/'
}
steps_graph = nx.DiGraph()
workflow = json.load(open('capdata/workflow.json'))
for step in workflow:
steps_graph.add_node(step['name'],step)
for x in step['dependencies']:
steps_graph.add_edge(x,step['name'])
rules = {}
for stepname in nx.topological_sort(steps_graph):
stepinfo = steps_graph.node[stepname]
rule = foradage.RECAST_Rule(stepinfo,workflow,rules,global_context)
rules[stepname] = rule
adage.rundag(g,rules.values(), track = True, backend = backend)
provgraph = nx.DiGraph()
for x in nx.topological_sort(g):
attr = g.node[x].copy()
attr.update(color = 'red',label = g.getNode(x).name)
provgraph.add_node(x,attr)
nodeinfo = g.getNode(x).task.node
if 'used_inputs' in nodeinfo:
for k,inputs_from_node in nodeinfo['used_inputs'].iteritems():
for one in inputs_from_node:
depname = 'output_{}_{}_{}'.format(k,one[0],one[1])
provgraph.add_edge(depname,x)
else:
for pre in g.predecessors(x):
provgraph.add_edge(pre,x)
for k,v in g.getNode(x).result_of()['RECAST_metadata']['outputs'].iteritems():
for i,y in enumerate(v):
name = 'output_{}_{}_{}'.format(g.getNode(x).task.node['name'],k,i)
provgraph.add_node(name,{'shape':'box','label':'{}_{}'.format(k,i),'color':'blue'})
provgraph.add_edge(x,name)
nx.write_dot(provgraph,'workflow_instance.dot')
subprocess.call(['dot','-Tpdf','workflow_instance.dot'], stdout = open('workflow_instance.pdf','w'))
nx.write_dot(steps_graph,'steps.dot')
subprocess.call(['dot','-Tpdf','steps.dot'], stdout = open('steps.pdf','w'))
def test_topological_sort2(self):
DG = nx.DiGraph({1: [2], 2: [3], 3: [4],
4: [5], 5: [1], 11: [12],
12: [13], 13: [14], 14: [15]})
assert_raises(nx.NetworkXUnfeasible, consume, nx.topological_sort(DG))
assert_false(nx.is_directed_acyclic_graph(DG))
DG.remove_edge(1, 2)
consume(nx.topological_sort(DG))
assert_true(nx.is_directed_acyclic_graph(DG))
def __init__(self, equations):
# Everythings must be an equation
self.equations = equations
self.equations_by_name = {}
for eq in equations:
if not isinstance(eq, Equation):
raise RuntimeError("Non Equation found.")
if eq.name in equations:
raise RuntimeError("Equations names must be unique within a graph")
self.equations_by_name[eq.name] = eq
# Make a network from this. The first is the full network of both
# equations and variables (a bipartite graph). The second is just the
# network of causal variables (the project of the bipartite graph).
full_network = nx.DiGraph()
causal_network = nx.DiGraph()
for p in equations:
for i in p.inputs:
full_network.add_edge(i, p)
for o in p.outputs:
full_network.add_edge(p, o)
for i in p.inputs:
for o in p.outputs:
causal_network.add_edge(i, o)
self.full_network = full_network
self.causal_network = causal_network
# Nodes are either inputs, outputs, or inner
self.inputs = set()
self.outputs = set()
self.inner = set()
for n in self.causal_network.nodes():
preds = self.full_network.predecessors(n)
sucs = self.full_network.successors(n)
if not preds:
self.inputs.add(n)
if not sucs:
self.outputs.add(n)
if preds and sucs:
self.inner.add(n)
# Sort all nodes into topological order. This allows us to calculate
# the probabilities across the nodes in the right order, so that the
# inputs for each player are always calculated in time (by previous
# equations).
self.ordered_variables = nx.topological_sort(self.causal_network)
self.ordered_nodes = nx.topological_sort(self.full_network)
self.graphviz_prettify(self.full_network)
self.graphviz_prettify(self.causal_network)
def add_edge(self, u, v, attr_dict=None, **attr):
"""Add an edge between u and v.
The nodes u and v will be automatically added if they are
not already in the graph.
Edge attributes can be specified with keywords or by providing
a dictionary with key/value pairs. See examples below.
Ensures no cycle are introduced in the graph.
Parameters
----------
u,v : nodes
Nodes can be, for example, strings or numbers.
Nodes must be hashable (and not None) Python objects.
attr_dict : dictionary, optional (default= no attributes)
Dictionary of edge attributes. Key/value pairs will
update existing data associated with the edge.
attr : keyword arguments, optional
Edge data (or labels or objects) can be assigned using
keyword arguments.
"""
# set up attribute dict
if attr_dict is None:
attr_dict = attr
else:
try:
attr_dict.update(attr)
except AttributeError:
raise NetworkXError("The attr_dict argument must be a dictionary.")
# add nodes
if u not in self.succ:
self.succ[u] = {}
self.pred[u] = {}
self.node[u] = {}
if v not in self.succ:
self.succ[v] = {}
self.pred[v] = {}
self.node[v] = {}
# add the edge
datadict = self.adj[u].get(v, {})
datadict.update(attr_dict)
self.succ[u][v] = datadict
self.pred[v][u] = datadict
# make sure it's a DAG
try:
networkx.topological_sort(self)
except networkx.NetworkXUnfeasible:
self.remove_edge(u, v)
raise networkx.NetworkXUnfeasible("Graph contains a cycle! DAG is acyclic!")
def test_topological_sort1(self):
DG=nx.DiGraph()
DG.add_edges_from([(1,2),(1,3),(2,3)])
assert_equal(nx.topological_sort(DG),[1, 2, 3])
assert_equal(nx.topological_sort_recursive(DG),[1, 2, 3])
DG.add_edge(3,2)
assert_raises(nx.NetworkXUnfeasible, nx.topological_sort, DG)
assert_raises(nx.NetworkXUnfeasible, nx.topological_sort_recursive, DG)
DG.remove_edge(2,3)
assert_equal(nx.topological_sort(DG),[1, 3, 2])
assert_equal(nx.topological_sort_recursive(DG),[1, 3, 2])
def __delitem__(self, key):
#print "__delitem__(" + str(key) + ")__"
#a special case: self
if key in ["self", "key", "value", "col", "row"]:
raise KeyError
# if we do not have the key, then we leave
if key not in self.state:
raise KeyError
# we remove the key
#self.G.remove_node(key)
del self.formulas[key]
del self.values[key]
#del self.mode[key]
#del self.state[key]
# we call the callbacks
for i in self.callbacks:
try:
i("delete", key)
except Exception as e:
print "callback delete " + key
print "callback :=" + str(i)
print "error: " + str(e)
pass
for i in self.named_callbacks:
try:
self.__getitem__(i)(self, "delete", key)
except Exception as e:
print "callback delete " + key
print "callback :=" + str(i)
print "error: " + str(e)
pass
# we mark all successor as dirty
for i in nx.topological_sort(self.G, [key]):
if i <> key:
#print str(key) + " -> " + str(i)
self.state[i] = 0
self.update(i)
# if there is not successor, we remove the node, and mode and state
if nx.topological_sort(self.G, [key]) == [key]:
self.G.remove_node(key)
del self.mode[key]
del self.state[key]
def _nodes(self, order=None):
"""
returns all nodes in the graph
"""
if order is None:
node_names = self.name_to_node.keys()
elif order == "architecture":
node_names = nx.topological_sort(self.architectural_tree)
elif order == "computation":
node_names = nx.topological_sort(self.computation_graph)
else:
raise ValueError("Unknown order: %s" % order)
# make sure that all of the original nodes are returned
assert set(self.name_to_node.keys()) == set(node_names)
return [self.name_to_node[name] for name in node_names]
def order_tables(dbdir_str):
"""
Sorts tables in constraint order, primary tables first
Constructs a constraint graph for the tables in the supplied directory,
sorts it in topological order, and returns the table names in that
order in a list.
"""
graph = nx.DiGraph()
for file_str in os.listdir(dbdir_str):
if file_str.endswith(".table"):
add_table_to_graph(dbdir_str, file_str, graph)
ordered_tables = None
try:
ordered_tables = nx.topological_sort(graph)
except nx.NetworkXUnfeasible:
print >> sys.stderr, "Tables and their constraints do not form a DAG"
# The NetworkX package does not include a function that finds cycles
# in a directed graph. Lacking that, report the undirected cycles.
# Since our tables currently contain no cycles other than self-loops,
# no need to implement a directed cycle finder.
cycles = nx.cycle_basis(nx.Graph(graph))
print >> sys.stderr, "Found possible cycles:"
print >> sys.stderr, cycles
sys.exit(1)
return ordered_tables
def binarize_dag(g,
vertex_weight_key,
edge_weight_key,
dummy_node_name_prefix='d_'):
g = g.copy() # be functional
dummy_node_counter = 1
for u in nx.topological_sort(g):
nbs = g.neighbors(u)
while len(nbs) > 2:
for p_1, p_2 in chunks(nbs, 2):
v = "{}{}".format(
dummy_node_name_prefix,
dummy_node_counter
)
g.add_node(v)
g.node[v]['dummy'] = True
g.add_edge(u, v)
g.add_edges_from([(v, p_1),
(v, p_2)])
g.node[v][vertex_weight_key] = 0
g[v][p_1][edge_weight_key] = g[u][p_1][edge_weight_key]
g[v][p_2][edge_weight_key] = g[u][p_2][edge_weight_key]
g[u][v][edge_weight_key] = 0
g.remove_edges_from([(u, p_1), (u, p_2)])
dummy_node_counter += 1
nbs = g.neighbors(u)
return g
def get_v_to_subtree_probs(nstates, G_dag, edge_to_P, leaf_to_state_index):
"""
Get partial likelihoods on an arbitrarily rooted tree.
Use a Felsenstein-like algorithm
to compute subtree probs given a subtree root state.
This depends on the rooted tree structure, the edge_to_P map,
and the states of the alignment column at the leaves.
@param nstates: number of states
@param G_dag: arbitrarily rooted genetic history with branch lengths
@param edge_to_P: (a, b) to transition matrix
@param leaf_to_state_index: alignment column information
@return: map from a vertex to a vector of state-conditioned subtree probs
"""
v_to_subtree_probs = {}
for a in reversed(nx.topological_sort(G_dag)):
successors = G_dag.successors(a)
if successors:
subtree_probs = np.ones(nstates, dtype=float)
for state_index in range(nstates):
for b in successors:
P = edge_to_P[a, b]
p = np.dot(P[state_index], v_to_subtree_probs[b])
subtree_probs[state_index] *= p
else:
state_index = leaf_to_state_index[a]
subtree_probs = np.zeros(nstates, dtype=float)
subtree_probs[state_index] = 1
v_to_subtree_probs[a] = subtree_probs
return v_to_subtree_probs
def _dconec(conec, inno):
"""
Return positions of edges (in conec) of graphs for
derivative calculation, all packed in one list (dconecno). Additionaly
beginings of each graph in this list is returned (dconecmk)
"""
dgraphs = []; dconecno = []; dconecmk = [0]
for idx, i in enumerate(inno):
dgraph = NX.DiGraph()
dgraph.add_edges_from(conec)
dgraph = _dependency(dgraph, i)
dsnodes = NX.topological_sort(dgraph)
for dnode in dsnodes:
try: # This is for networkx-0.3x
for dedge in dgraph.in_edges(dnode):
idx = conec.index(dedge) + 1
dconecno.append(idx)
except: # This is for new networkx-0.99
for dedge in (dgraph.reverse(copy=True)).edges(dnode):
dedge = (dedge[1], dedge[0]) #!!!
idx = conec.index(dedge) + 1
dconecno.append(idx)
dgraphs.append(dgraph)
dconecmk.append(len(dconecno))
return dgraphs, dconecno, dconecmk
def get_levels(G):
levels = {}
for n in nx.topological_sort(G):
levels[n] = 0
for pred in G.predecessors_iter(n):
levels[n] = max(levels[n], levels[pred]+1)
return levels
def _iterable_nodes(graph_in):
"""Returns the iterable nodes in the given graph and their join
dependencies.
The nodes are ordered as follows:
- nodes without an itersource precede nodes with an itersource
- nodes without an itersource are sorted in reverse topological order
- nodes with an itersource are sorted in topological order
This order implies the following:
- every iterable node without an itersource is expanded before any
node with an itersource
- every iterable node without an itersource is expanded before any
of it's predecessor iterable nodes without an itersource
- every node with an itersource is expanded before any of it's
successor nodes with an itersource
Return the iterable nodes list
"""
nodes = nx.topological_sort(graph_in)
inodes = [node for node in nodes if node.iterables is not None]
inodes_no_src = [node for node in inodes if not node.itersource]
inodes_src = [node for node in inodes if node.itersource]
inodes_no_src.reverse()
return inodes_no_src + inodes_src
def _ffconec(conec):
"""
Checks if conec is acyclic, sorts it if necessary and returns tuple:
(conec, inno, hidno, outno) where:
conec - sorted input connectivity
inno/hidno/outno - lists of input/hidden/ouput units
"""
if len(conec) == 0: raise ValueError("Empty connectivity list")
graph = NX.DiGraph()
graph.add_edges_from(conec)
snodes = NX.topological_sort(graph)
if not snodes:
raise TypeError("Network has cycles.")
else:
conec = []; inno = []; hidno = []; outno = []
for node in snodes:
try: # This is for networkx-0.3x
ins = graph.in_edges(node)
outs = graph.out_edges(node)
except: # This is for new networkx-0.99
ins = (graph.reverse(copy=True)).edges(node)
ins = [(v,u) for (u,v) in ins] # reversing back!
outs = graph.edges(node)
if not ins and node != 0 : # biases handling
inno += [node]
else:
conec += ins #Maybe + [(0,node)] i.e. bias
if not outs: outno += [node]
else:
if node != 0: hidno += [node] #bias handling again
return graph, conec, inno, hidno, outno
def _bconec(conec, inno):
"""
Returns positions of edges of reversed graph in conec (for backprop).
Conec is assumed to be acyclic.
"""
bgraph = NX.DiGraph()
bgraph.add_edges_from(conec)
bgraph = bgraph.reverse()
try: # This is for networkx-0.3x
bgraph.delete_nodes_from(inno)
try: bgraph.delete_node(0) #handling biases
except: pass
except: # This is for networkx >= 0.99
bgraph.remove_nodes_from(inno)
try: bgraph.remove_node(0) #handling biases
except: pass
bsnodes = NX.topological_sort(bgraph)
bconecno = []
for bnode in bsnodes:
try: # This is for networkx-0.3x
for bedge in bgraph.in_edges(bnode):
edge = (bedge[1], bedge[0])
idx = conec.index(edge) + 1
bconecno.append(idx)
except: # This is for new networkx-0.99
for bedge in (bgraph.reverse(copy=True)).edges(bnode):
edge = bedge #(bedge[1], bedge[0])
idx = conec.index(edge) + 1
bconecno.append(idx)
return bgraph, bconecno
def test_bidirectional(self):
"""
m1 <-> m2 -> m3
"""
self.g.add_edge('m1','m2')
self.g.add_edge('m2','m3')
self.g.add_edge('m3','m2')
self.g.add_edge('m3','m4')
# remove any models that done have links
#for
# determine cycles
cycles = n.recursive_simple_cycles(self.g)
for cycle in cycles:
# remove edges that form cycles
self.g.remove_edge(cycle[0],cycle[1])
# perform toposort
order = n.topological_sort(self.g)
# re-add bidirectional dependencies (i.e. cycles)
for cycle in cycles:
# find index of inverse link
for i in xrange(0,len(order)-1):
if order[i] == cycle[1] and order[i+1] == cycle[0]:
order.insert(i+2, cycle[1])
order.insert(i+3,cycle[0])
break
self.assertTrue(''.join(order) == 'm1m2m3m2m3m4')
def __init__(self, model, dt=0.001, seed=None, builder=None):
if builder is None:
# By default, we'll use builder.Builder and copy the model.
builder = Builder(copy=True)
# Call the builder to build the model
self.model = builder(model, dt)
# Note: seed is not used right now, but one day...
if seed is None:
seed = self.model._get_new_seed() # generate simulator seed
# -- map from Signal.base -> ndarray
self._sigdict = SignalDict(__time__=np.asarray(0.0, dtype=np.float64))
for op in self.model.operators:
op.init_sigdict(self._sigdict, self.model.dt)
self.dg = self._init_dg()
self._step_order = [node
for node in nx.topological_sort(self.dg)
if hasattr(node, 'make_step')]
self._steps = [node.make_step(self._sigdict, self.model.dt)
for node in self._step_order]
self.n_steps = 0
self.probe_outputs = dict((probe, []) for probe in self.model.probes)
def test_topological_sort3(self):
DG = nx.DiGraph()
DG.add_edges_from([(1, i) for i in range(2, 5)])
DG.add_edges_from([(2, i) for i in range(5, 9)])
DG.add_edges_from([(6, i) for i in range(9, 12)])
DG.add_edges_from([(4, i) for i in range(12, 15)])
def validate(order):
ok_(isinstance(order, list))
assert_equal(set(order), set(DG))
for u, v in combinations(order, 2):
assert_false(nx.has_path(DG, v, u))
validate(list(nx.topological_sort(DG)))
DG.add_edge(14, 1)
assert_raises(nx.NetworkXUnfeasible, consume, nx.topological_sort(DG))
def dag(recipe_folder, config, packages="*", format='gml', hide_singletons=False):
"""
Export the DAG of packages to a graph format file for visualization
"""
dag, name2recipes = graph.build(utils.get_recipes(recipe_folder, "*"), config)
if packages != "*":
dag = graph.filter(dag, packages)
if hide_singletons:
for node in nx.nodes(dag):
if dag.degree(node) == 0:
dag.remove_node(node)
if format == 'gml':
nx.write_gml(dag, sys.stdout.buffer)
elif format == 'dot':
write_dot(dag, sys.stdout)
elif format == 'txt':
subdags = sorted(map(sorted, nx.connected_components(dag.to_undirected())))
subdags = sorted(subdags, key=len, reverse=True)
singletons = []
for i, s in enumerate(subdags):
if len(s) == 1:
singletons += s
continue
print("# subdag {0}".format(i))
subdag = dag.subgraph(s)
recipes = [
recipe for package in nx.topological_sort(subdag)
for recipe in name2recipes[package]]
print('\n'.join(recipes) + '\n')
if not hide_singletons:
print('# singletons')
recipes = [recipe for package in singletons for recipe in
name2recipes[package]]
print('\n'.join(recipes) + '\n')
def gotermSummarization(self, P_geneList, P_value, P_minProteinNo): #use P_GotermNumber to summarizate a given list of proteins #Topologically sort at first self.node_Mapping_ProteinList_PV2(P_geneList) self.symplify_Goterm_Structure() #Topologically sort all node, the first element is a leaf and the last element is the root New_Top_sort_nodes = networkx.topological_sort(self) #go through all nodes in topological order from leaves to the root. for nodeTp_child in New_Top_sort_nodes: nodePV = self.node[nodeTp_child][self.PV] nodeSize = len(self.node[nodeTp_child][self.mapping]) #if the current node's pv or size is not constrained by the setting conditions, merge this node to the most close parent node. if nodePV>P_value or nodeSize<P_minProteinNo: L_parrents = self.successors(nodeTp_child) L_MinWeight = 100000; L_MinPa = 'none' for nodetp in L_parrents: if self.edge[nodeTp_child][nodetp]['weight']<L_MinWeight: L_MinWeight = self.edge[nodeTp_child][nodetp]['weight'] L_MinPa = nodetp if L_MinPa == 'none': pass else: self.merge_nodes2(nodeTp_child,L_MinPa) return self.printResult()
def submit_jobs(view, G, jobs):
"""Submit jobs via client where G describes the time dependencies."""
results = {}
for node in nx.topological_sort(G):
with view.temp_flags(after=[ results[n] for n in G.predecessors(node) ]):
results[node] = view.apply(jobs[node])
return results
def _node_iter(G, g_inputs):
"""
Run a graph
G : nx.DiGraph
Nodes are tools, edges contain attribute 'link' which
is a list of A -> B pairs for connecting the inputs
and outputs
g_inputs : dict
Global inputs to tools. Keyed on tool
instances, values are dicts keyed on name
of input values
"""
# sort the tools by topological order so all needed inputs are
# available
for job in nx.topological_sort(G):
# get input edges
input_edges = G.in_edges(job)
in_link_info = dict()
# construct dict of useful information
for parent, child in input_edges:
print(parent, child)
in_link_info[parent] = G[parent][child]['links']
# get output edges
output_edges = G.out_edges(job)
out_link_info = dict()
# construct dict of useful information
for parent, child in output_edges:
out_link_info[child] = G[parent][child]['links']
# pull the tool args out of the global dict
args = g_inputs[job]
# push work off to helper function
yield (job, in_link_info, out_link_info, args)
def _resolve_update_list(self, changed_properties):
"""
Returns a list of all plasma models which are affected by the
changed_modules due to there dependency in the
the plasma_graph.
Parameters
----------
changed_modules: ~list
all modules changed in the plasma
Returns
-------
: ~list
all affected modules.
"""
descendants_ob = []
for plasma_property in changed_properties:
node_name = self.outputs_dict[plasma_property].name
descendants_ob += nx.descendants(self.graph, node_name)
descendants_ob = list(set(descendants_ob))
sort_order = nx.topological_sort(self.graph)
descendants_ob.sort(key=lambda val: sort_order.index(val))
logger.debug("Updating modules in the following order:".format("->".join(descendants_ob)))
return descendants_ob
def compute_builds(path,
base_name,
git_rev=None,
stop_rev=None,
folders=None,
matrix_base_dir=None,
steps=0,
max_downstream=5,
test=False,
public=True,
output_dir='../output',
output_folder_label='git',
config_overrides=None,
**kw):
if not git_rev and not folders:
raise ValueError(
"Either git_rev or folders list are required to know what to compute"
)
checkout_rev = stop_rev or git_rev
folders = folders
path = path.replace('"', '')
if not folders:
folders = git_changed_recipes(git_rev, stop_rev, git_root=path)
if not folders:
print(
"No folders specified to build, and nothing changed in git. Exiting."
)
return
matrix_base_dir = matrix_base_dir or path
# clean up quoting from concourse template evaluation
matrix_base_dir = matrix_base_dir.replace('"', '')
repo_commit = ''
git_identifier = ''
if checkout_rev:
with checkout_git_rev(checkout_rev, path):
git_identifier = _get_current_git_rev(path)
task_graph = collect_tasks(path,
folders=folders,
steps=steps,
max_downstream=max_downstream,
test=test,
matrix_base_dir=matrix_base_dir)
try:
repo_commit = _get_current_git_rev(path)
except subprocess.CalledProcessError:
repo_commit = 'master'
else:
task_graph = collect_tasks(path,
folders=folders,
steps=steps,
max_downstream=max_downstream,
test=test,
matrix_base_dir=matrix_base_dir)
with open(os.path.join(matrix_base_dir, 'config.yml')) as src:
data = yaml.load(src)
data['recipe-repo-commit'] = repo_commit
if config_overrides:
data.update(config_overrides)
plan = graph_to_plan_with_jobs(os.path.abspath(path),
task_graph,
commit_id=repo_commit,
matrix_base_dir=matrix_base_dir,
config_vars=data,
public=public)
output_dir = output_dir.format(base_name=base_name,
git_identifier=git_identifier)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
with open(os.path.join(output_dir, 'plan.yml'), 'w') as f:
yaml.dump(plan, f, default_flow_style=False)
# expand folders to include any dependency builds or tests
if not os.path.isabs(path):
path = os.path.normpath(os.path.join(os.getcwd(), path))
for fn in glob.glob(os.path.join(output_dir, 'output_order*')):
os.remove(fn)
last_recipe_dir = None
for node in nx.topological_sort(task_graph, reverse=True):
meta = task_graph.node[node]['meta']
if meta.meta_path:
recipe = os.path.dirname(meta.meta_path)
else:
recipe = meta.get('extra', {}).get('parent_recipe', {})
assert recipe, ("no parent recipe set, and no path associated "
"with this metadata")
# make recipe path relative
recipe = recipe.replace(path + '/', '')
# copy base recipe into a folder named for this node
out_folder = os.path.join(output_dir, node)
if os.path.isdir(out_folder):
shutil.rmtree(out_folder)
shutil.copytree(os.path.join(path, recipe), out_folder)
# write the conda_build_config.yml for this particular metadata into that recipe
# This should sit alongside meta.yaml, where conda-build will be able to find it
with open(os.path.join(out_folder, 'conda_build_config.yaml'),
'w') as f:
yaml.dump(meta.config.squished_variants,
f,
default_flow_style=False)
order_fn = 'output_order_' + task_graph.node[node]['worker']['label']
with open(os.path.join(output_dir, order_fn), 'a') as f:
f.write(node + '\n')
recipe_dir = os.path.dirname(recipe) if os.sep in recipe else recipe
if not last_recipe_dir or last_recipe_dir != recipe_dir:
order_recipes_fn = 'output_order_recipes_' + task_graph.node[node][
'worker']['label']
with open(os.path.join(output_dir, order_recipes_fn), 'a') as f:
f.write(recipe_dir + '\n')
last_recipe_dir = recipe_dir
def get_schedule(self, simulation):
schedule = {host: [] for host in simulation.hosts}
graph = simulation.get_task_graph()
for task in networkx.topological_sort(graph):
schedule[random.choice(simulation.hosts)].append(task)
return schedule
def _generate_flatgraph(self):
"""Generate a graph containing only Nodes or MapNodes
"""
import networkx as nx
logger.debug('expanding workflow: %s', self)
nodes2remove = []
if not nx.is_directed_acyclic_graph(self._graph):
raise Exception(('Workflow: %s is not a directed acyclic graph '
'(DAG)') % self.name)
nodes = list(nx.topological_sort(self._graph))
for node in nodes:
logger.debug('processing node: %s', node)
if isinstance(node, Workflow):
nodes2remove.append(node)
# use in_edges instead of in_edges_iter to allow
# disconnections to take place properly. otherwise, the
# edge dict is modified.
# dj: added list() for networkx ver.2
for u, _, d in list(
self._graph.in_edges(nbunch=node, data=True)):
logger.debug('in: connections-> %s', to_str(d['connect']))
for cd in deepcopy(d['connect']):
logger.debug("in: %s", to_str(cd))
dstnode = node._get_parameter_node(cd[1], subtype='in')
srcnode = u
srcout = cd[0]
dstin = cd[1].split('.')[-1]
logger.debug('in edges: %s %s %s %s', srcnode, srcout,
dstnode, dstin)
self.disconnect(u, cd[0], node, cd[1])
self.connect(srcnode, srcout, dstnode, dstin)
# do not use out_edges_iter for reasons stated in in_edges
# dj: for ver 2 use list(out_edges)
for _, v, d in list(
self._graph.out_edges(nbunch=node, data=True)):
logger.debug('out: connections-> %s', to_str(d['connect']))
for cd in deepcopy(d['connect']):
logger.debug("out: %s", to_str(cd))
dstnode = v
if isinstance(cd[0], tuple):
parameter = cd[0][0]
else:
parameter = cd[0]
srcnode = node._get_parameter_node(parameter,
subtype='out')
if isinstance(cd[0], tuple):
srcout = list(cd[0])
srcout[0] = parameter.split('.')[-1]
srcout = tuple(srcout)
else:
srcout = parameter.split('.')[-1]
dstin = cd[1]
logger.debug('out edges: %s %s %s %s', srcnode, srcout,
dstnode, dstin)
self.disconnect(node, cd[0], v, cd[1])
self.connect(srcnode, srcout, dstnode, dstin)
# expand the workflow node
# logger.debug('expanding workflow: %s', node)
node._generate_flatgraph()
for innernode in node._graph.nodes():
innernode._hierarchy = '.'.join(
(self.name, innernode._hierarchy))
self._graph.add_nodes_from(node._graph.nodes())
self._graph.add_edges_from(node._graph.edges(data=True))
if nodes2remove:
self._graph.remove_nodes_from(nodes2remove)
logger.debug('finished expanding workflow: %s', self)
def get_target_topic(self):
return nx.topological_sort(self.graph)[-1]
def set_max_size(g, max_size):
unary_graph = nx.DiGraph()
for x in g.nonterminals:
unary_graph.add_node(x)
for lhs, rules in g.by_lhs.items():
for rule in rules:
if len(rule.cfg_rhs
) == 1 and rule.cfg_rhs[0] in g.nonterminals:
unary_graph.add_edge(rule.lhs,
rule.cfg_rhs[0],
weight=rule.prob)
try:
topological = nx.topological_sort(nx.DiGraph(unary_graph),
reverse=True)
unary_matrix = None
except nx.NetworkXUnfeasible:
topological = list(g.nonterminals)
# unary_matrix[i][j] == 1 means there is a rule i->j
unary_matrix = np.array(
nx.to_numpy_matrix(unary_graph, topological))
# find infinite summation over chains of unary rules (at least one long)
try:
unary_matrix = np.dot(
unary_matrix,
np.array(
np.linalg.inv(np.eye(len(topological)) -
unary_matrix)))
except np.linalg.LinAlgError as e:
raise np.linalg.LinAlgError(
e.message + " (cycle of unary rules with weight >= 1)")
nt_to_index = {x: i for i, x in enumerate(topological)}
alpha = np.empty((len(topological), max_size + 1))
alpha.fill(-np.inf)
for size in range(1, max_size + 1):
for lhs_i, lhs in enumerate(topological):
for rule in g.by_lhs[lhs]:
if unary_matrix is not None:
# we'll do unary rules later
if len(rule.cfg_rhs
) == 1 and rule.cfg_rhs[0] in g.nonterminals:
continue
nts = [
nt_to_index[x] for x in rule.cfg_rhs
if x in g.nonterminals
]
n = size - (len(rule.cfg_rhs) - len(nts)
) # total size available for nonterminals
if len(nts) == 0:
if n != 0:
continue
p = 0.
elif len(nts) == 1:
p = alpha[nts[0], n]
elif len(nts) == 2:
if n < 2:
continue
p = np.logaddexp.reduce([
alpha[nts[0], k] + alpha[nts[1], n - k]
for k in g.splits(n)
])
else:
raise ValueError("more than two nonterminals in rhs")
with np.errstate(invalid='ignore'):
alpha[lhs_i,
size] = np.logaddexp(alpha[lhs_i, size],
np.log(rule.prob) + p)
# Apply unary rules
# If we weren't in log-space, this would be:
# alpha[:,size] = unary_matrix * alpha[:,size]
if unary_matrix is not None:
lz = np.max(alpha[:, size])
# the reason we made unary_matrix be 1+ applications of unary
# rules is just in case of underflow here
alpha[:, size] = np.logaddexp(
alpha[:, size],
np.log(np.dot(unary_matrix, np.exp(alpha[:, size] - lz))) +
lz)
g.alpha = alpha
g.topological = topological
def generate_reference(name, chain):
"""Generates a simple, unoptimized SDFG to run on the CPU, for verification
purposes."""
sdfg = SDFG(name)
for k, v in chain.constants.items():
sdfg.add_constant(k, v["value"], dace.data.Scalar(v["data_type"]))
(dimensions_to_skip, shape, vector_length, parameters, iterators,
memcopy_indices, memcopy_accesses) = _generate_init(chain)
prev_state = sdfg.add_state("init")
# Throw vectorization in the bin for the reference code
vector_length = 1
shape = tuple(map(int, shape))
input_shapes = {} # Maps inputs to their shape tuple
for node in chain.graph.nodes():
if isinstance(node, Input) or isinstance(node, Output):
if isinstance(node, Input):
for output in node.outputs.values():
pars = tuple(
output["input_dims"]
) if "input_dims" in output and output[
"input_dims"] is not None else tuple(parameters)
arr_shape = tuple(s for s, p in zip(shape, parameters)
if p in pars)
input_shapes[node.name] = arr_shape
break
else:
raise ValueError("No outputs found for input node.")
else:
arr_shape = shape
if len(arr_shape) > 0:
try:
sdfg.add_array(node.name, arr_shape, node.data_type)
except NameError:
sdfg.data(
node.name).access = dace.dtypes.AccessType.ReadWrite
else:
sdfg.add_symbol(node.name, node.data_type)
for link in chain.graph.edges(data=True):
name = link[0].name
if name not in sdfg.arrays and name not in sdfg.symbols:
sdfg.add_array(name, shape, link[0].data_type, transient=True)
input_shapes[name] = tuple(shape)
input_iterators = {
k: tuple("0:{}".format(s) for s in v)
for k, v in input_shapes.items()
}
# Enforce dependencies via topological sort
for node in nx.topological_sort(chain.graph):
if not isinstance(node, Kernel):
continue
state = sdfg.add_state(node.name)
sdfg.add_edge(prev_state, state, dace.InterstateEdge())
(stencil_node, input_to_connector,
output_to_connector) = _generate_stencil(node, chain, shape,
dimensions_to_skip)
stencil_node.implementation = "CPU"
for field, connector in input_to_connector.items():
if len(input_iterators[field]) == 0:
continue # Scalar variable
# Outer memory read
read_node = state.add_read(field)
state.add_memlet_path(read_node,
stencil_node,
dst_conn=connector,
memlet=Memlet.simple(
field,
", ".join(input_iterators[field])))
for _, connector in output_to_connector.items():
# Outer write
write_node = state.add_write(node.name)
state.add_memlet_path(stencil_node,
write_node,
src_conn=connector,
memlet=Memlet.simple(
node.name, ", ".join("0:{}".format(s)
for s in shape)))
prev_state = state
return sdfg
def maximal_independant_longest_path_for_acyclic_graph(acyclic_graph,inv_mapping={}):
# Compute longest-path-acyclic
g_dir_trans_io = nx.DiGraph(acyclic_graph)
# After the cycle reduction, some nodes might have disappeared :
# This can happen in cases where we had a SCC but without inputs. Once this SCC is removed, some nodes might not have predecessors anymore
# We thus connect these nodes without pred back to the In node
for inp in [x for x,v in g_dir_trans_io.node.items() if ("In" in v) and (x!="In")]+[x for x,v in g_dir_trans_io.in_degree().items() if (v==0) and (x!="In")]:
g_dir_trans_io.add_edge("In",inp,attr_dict={"weight":0})
for inp in [x for x,v in g_dir_trans_io.node.items() if ("Out" in v) and (x!="Out")]+[x for x,v in g_dir_trans_io.out_degree().items() if (v==0) and (x!="Out")]:
g_dir_trans_io.add_edge(inp,"Out",attr_dict={"weight":0})
nodes_in_order = nx.topological_sort(g_dir_trans_io)
# The sort should start and end with resp. In and Out
assert(nodes_in_order[0]=="In")
assert(nodes_in_order[-1]=="Out")
# Update each node attributes with the accumulated length from the beginning
# Q? Should we store each path separately or a greedy alg is ok? trying greedy
# Q? How to deal with mulitple equivalent paths ?
for n in g_dir_trans_io:
g_dir_trans_io.node[n]['in_longest']="False"
for src,tgt in g_dir_trans_io.edges():
g_dir_trans_io[src][tgt]["in_longest"]="False"
all_longest_path_trans_io = []
nodes_in_any_longest_path = set()
inputs_nodes = [k for k,v in g_dir_trans_io.in_degree().items() if v==0]
inputs_nodes.extend([x for x,v in g_dir_trans_io.node.items() if "In" in v])
outputs_nodes = [k for k,v in g_dir_trans_io.out_degree().items() if v==0]
outputs_nodes.extend([x for x,v in g_dir_trans_io.node.items() if "Out" in v])
# Todo: Account for contig ID in longest path
for long_path_index in range(0,2000):
# Clear all longest path info
print "longest path iteration",long_path_index
# print nodes_in_any_longest_path
for n in g_dir_trans_io.node.keys():
if "longest_path" in g_dir_trans_io[n]:
del g_dir_trans_io[n]['longest_path']
#Init
g_dir_trans_io.node['In']['longest_path']=[0,[]]
for n in nodes_in_order[1:]:
contig = g_dir_trans_io.node[n].get("contig",n)
if contig in nodes_in_any_longest_path:
continue
pred = [x for x in g_dir_trans_io.predecessors(n) if g_dir_trans_io.node[x].get("contig",x) not in nodes_in_any_longest_path]
if len(pred)==0:
# This can happen in cases where we had a SCC but without inputs. Once this SCC is removed, some nodes might not have predecessors anymore
longest_path=[0,[]]
else:
longest_path= [(g_dir_trans_io.node[pred[0]]['longest_path'][0] + g_dir_trans_io[pred[0]][n]['weight']),\
pred[0]]
for p in pred:
if (g_dir_trans_io.node[p]['longest_path'][0]+ g_dir_trans_io[p][n]['weight']) > longest_path[0]:
longest_path= [(g_dir_trans_io.node[p]['longest_path'][0]+ g_dir_trans_io[p][n]['weight']),p]
g_dir_trans_io.node[n]['longest_path']=longest_path
# Rebuild the longest path, starting from "Out" up to "In" in predecessor order
current_node = "Out"
longest_path_trans_io = []
while (current_node != "In") and current_node!=[]:
longest_path_trans_io.append((current_node,g_dir_trans_io.node[current_node]["longest_path"][0]))
current_node=g_dir_trans_io.node[current_node]['longest_path'][1]
if current_node==[]:
break
longest_path_trans_io.append((current_node,g_dir_trans_io.node[current_node]["longest_path"][0]))
longest_path_trans_io.reverse()
print longest_path_trans_io
print [g_dir_trans_io.node[x[0]].get("contig",x[0]) for x in longest_path_trans_io]
if len(longest_path_trans_io)<=2: # only in and out?
break
all_longest_path_trans_io.append(longest_path_trans_io)
# Mark this path in the graph
g_dir_trans_io.node['Out']['in_longest']="True"
for i in range(0, len(longest_path_trans_io)-1):
n = longest_path_trans_io[i][0]
g_dir_trans_io.node[n]['in_longest']=long_path_index
g_dir_trans_io[longest_path_trans_io[i][0]][longest_path_trans_io[i+1][0]]['in_longest']=long_path_index
# debug_here()
if (n not in ["In","Out"]):
nodes_in_any_longest_path.add(g_dir_trans_io.node[n]['contig'])
# Trasnform the longest path into the original graph IDs
original_longest_paths=[]
for a_lp in all_longest_path_trans_io:
lp = [inv_mapping.get(x[0].split("@")[0],x[0].split("@")[0]) for x in a_lp]
original_longest_paths.append(lp)
return original_longest_paths
def _keep_fused_form(self, posPreferenceDicts):
# For a span A,B and external tokens C, such as A > B > C, we have to
# Make A the head of the span
# Attach C-level tokens to A
#Remove B-level tokens, which are the subtokens of the fused form della: de la
if self.graph["multi_tokens"] == {}:
return
spanheads = []
spanhead_fused_token_dict = {}
# This double iteration is overkill, one could skip the spanhead identification
# but in this way we avoid modifying the tree as we read it
for fusedform_idx in sorted(self.graph["multi_tokens"]):
fusedform_start, fusedform_end = self.graph["multi_tokens"][
fusedform_idx]["id"]
fuseform_span = list(range(fusedform_start, fusedform_end + 1))
spanhead = self._choose_spanhead_from_heuristics(
fuseform_span, posPreferenceDicts)
#if not spanhead:
# spanhead = self._choose_spanhead_from_heuristics(fuseform_span,posPreferenceDicts)
spanheads.append(spanhead)
spanhead_fused_token_dict[spanhead] = fusedform_idx
# try:
# order = list(nx.topological_sort(self))
# except nx.NetworkXUnfeasible:
# msg = 'Circular dependency detected between hooks'
# problem_graph = ', '.join(f'{a} -> {b}'
# for a, b in nx.find_cycle(self))
# print('nx.simple_cycles', list(nx.simple_cycles(self)))
# print(problem_graph)
# exit(0)
# for edge in list(nx.simple_cycles(self)):
# self.remove_edge(edge[0], edge[1])
self = remove_all_cycle(self)
bottom_up_order = [
x for x in nx.topological_sort(self) if x in spanheads
]
for spanhead in bottom_up_order:
fusedform_idx = spanhead_fused_token_dict[spanhead]
fusedform = self.graph["multi_tokens"][fusedform_idx]["form"]
fusedform_start, fusedform_end = self.graph["multi_tokens"][
fusedform_idx]["id"]
fuseform_span = list(range(fusedform_start, fusedform_end + 1))
if spanhead:
#Step 1: Replace form of head span (A) with fusedtoken form -- in this way we keep the lemma and features if any
self.nodes[spanhead]["form"] = fusedform
# 2- Reattach C-level (external dependents) to A
#print(fuseform_span,spanhead)
internal_dependents = set(fuseform_span) - set([spanhead])
external_dependents = [
nx.bfs_successors(self, x) for x in internal_dependents
]
for depdict in external_dependents:
for localhead in depdict:
for ext_dep in depdict[localhead]:
if ext_dep in self[localhead]:
deprel = self[localhead][ext_dep]["deprel"]
self.remove_edge(localhead, ext_dep)
self.add_edge(spanhead, ext_dep, deprel=deprel)
#3- Remove B-level tokens
for int_dep in internal_dependents:
self.remove_edge(self.head_of(int_dep), int_dep)
self.remove_node(int_dep)
#4 reconstruct tree at the very end
new_index_dict = {}
for new_node_index, old_node_idex in enumerate(sorted(self.nodes())):
new_index_dict[old_node_idex] = new_node_index
T = DependencyTree() # Transfer DiGraph, to replace self
for n in sorted(self.nodes()):
T.add_node(new_index_dict[n], self.nodes[n])
for h, d in self.edges():
T.add_edge(new_index_dict[h],
new_index_dict[d],
deprel=self[h][d]["deprel"])
#4A Quick removal of edges and nodes
self.__init__()
#4B Rewriting the Deptree in Self
# TODO There must a more elegant way to rewrite self -- self= T for instance?
for n in sorted(T.nodes()):
self.add_node(n, T.nodes[n])
for h, d in T.edges():
self.add_edge(h, d, T[h][d])
# 5. remove all fused forms form the multi_tokens field
self.graph["multi_tokens"] = {}
def apply(cls, stack, model, *args, **kwargs):
if 'path_to_feeder_file' in kwargs:
path_to_feeder_file = kwargs['path_to_feeder_file']
path_to_no_feeder_file = None
if 'path_to_no_feeder_file' in kwargs:
path_to_no_feeder_file = kwargs['path_to_no_feeder_file']
if 'compute_metrics' in kwargs:
compute_metrics = kwargs['compute_metrics']
else:
compute_metrics = False
if 'compute_kva_density_with_transformers' in kwargs:
compute_kva_density_with_transformers = kwargs[
'compute_kva_density_with_transformers']
else:
compute_kva_density_with_transformers = True
if compute_metrics:
if 'excel_output' in kwargs:
excel_output = kwargs['excel_output']
else:
raise ValueError('Missing output file name for excel')
if 'json_output' in kwargs:
json_output = kwargs['json_output']
else:
raise ValueError('Missing output file name for json')
#Open and read feeder.txt
with open(path_to_feeder_file, 'r') as f:
lines = f.readlines()
#Parse feeder.txt to have the feeder structure of the network
feeders = {}
substations = {}
substation_transformers = {}
for line in lines[1:]:
#Parse the line
node, sub, feed, sub_trans = map(lambda x: x.strip().lower(),
line.split(' '))
#If feeder is new, then add it to the feeders dict
if feed not in feeders:
#Initialize with a list holding the node
feeders[feed] = [node.lower().replace('.', '')]
#Othewise, just append the node
else:
feeders[feed].append(node.lower().replace('.', ''))
#Same thing for the substation
if feed not in substations:
substations[feed] = sub.lower().replace('.', '')
#Same thing for substation_transformers
if feed not in substation_transformers:
substation_transformers[feed] = sub.lower().replace('.', '')
if path_to_no_feeder_file is not None:
with open(path_to_no_feeder_file, 'r') as f:
lines = f.readlines()
for line in lines[1:]:
node, feed = map(lambda x: x.strip().lower(), line.split(' '))
if feed != 'mv-mesh':
if 'subtransmission' not in feeders:
feeders['subtransmission'] = [
node.lower().replace('.', '')
]
else:
feeders['subtransmission'].append(node.lower().replace(
'.', ''))
if 'subtransmission' not in substations:
substations['subtransmission'] = ''
#Create a network analyzer object
network_analyst = NetworkAnalyzer(model)
#Add the feeder information to the network analyzer
network_analyst.add_feeder_information(
list(feeders.keys()), list(feeders.values()), substations,
'') #TODO find a way to get the feeder type
#Split the network into feeders
network_analyst.split_network_into_feeders()
#Tag the objects
network_analyst.tag_objects()
#Set the names
network_analyst.model.set_names()
# Set reclosers. This algorithm finds to closest 1/3 of goabs to the feeder head (in topological order)
# without common ancestry. i.e. no recloser should be upstream of another recloser. If this is not possible,
# the number of reclosers is decreased
recloser_proportion = 0.33
all_goabs = {}
np.random.seed(0)
tmp_network = Network()
tmp_network.build(network_analyst.model, 'st_mat')
tmp_network.set_attributes(network_analyst.model)
tmp_network.remove_open_switches(network_analyst.model)
tmp_network.rebuild_digraph(network_analyst.model, 'st_mat')
sorted_elements = []
for element in nx.topological_sort(tmp_network.digraph):
sorted_elements.append(element)
for i in network_analyst.model.models:
if isinstance(
i, Line
) and i.is_switch is not None and i.is_switch and i.name is not None and 'goab' in i.name.lower(
):
is_open = False
for wire in i.wires:
if wire.is_open:
is_open = True
if is_open:
continue
if hasattr(
i, 'feeder_name'
) and i.feeder_name is not None and i.feeder_name != 'subtransmission':
if i.feeder_name in all_goabs:
all_goabs[i.feeder_name].append(i.name)
else:
all_goabs[i.feeder_name] = [i.name]
for key in list(all_goabs.keys()):
feeder_goabs_dic = {
} # Topological sorting done by node. This matches goabs to their end-node
for goab in all_goabs[key]:
feeder_goabs_dic[
model[goab].
to_element] = goab # shouldn't have multiple switches ending at the same node
feeder_goabs = []
feeder_goab_ends = []
for element in sorted_elements:
if element in feeder_goabs_dic:
feeder_goabs.append(feeder_goabs_dic[element])
feeder_goab_ends.append(element)
connectivity_matrix = [[False for i in range(len(feeder_goabs))]
for j in range(len(feeder_goabs))]
for i in range(len(feeder_goabs)):
recloser1 = feeder_goab_ends[i]
for j in range(i + 1, len(feeder_goabs)):
recloser2 = feeder_goab_ends[j]
if recloser2 == recloser1:
continue
connected = nx.has_path(tmp_network.digraph, recloser1,
recloser2)
connectivity_matrix[i][j] = connected
if connected:
connectivity_matrix[j][i] = connected
selected_goabs = []
num_goabs = int(len(feeder_goabs) * float(recloser_proportion))
finished = False
if num_goabs == 0:
finished = True
while not finished:
for i in range(len(feeder_goabs)):
current_set = set([i])
for j in range(i + 1, len(feeder_goabs)):
skip_this_one = False
for k in current_set:
if connectivity_matrix[j][
k]: #i.e. see if the candidate node has common anything upstream or downstream
skip_this_one = True
break
if skip_this_one:
continue
current_set.add(j)
if len(current_set) == num_goabs:
break
if len(current_set) == num_goabs:
finished = True
for k in current_set:
selected_goabs.append(feeder_goabs[k])
break
if not finished:
num_goabs -= 1
#selected_goabs = np.random.choice(feeder_goabs,int(len(feeder_goabs)*float(recloser_proportion)))
for recloser in selected_goabs:
network_analyst.model[recloser].is_switch = False
network_analyst.model[recloser].is_recloser = True
if network_analyst.model[recloser].wires is not None:
for wire in network_analyst.model[recloser].wires:
wire.is_switch = False
wire.is_recloser = True
network_analyst.model[recloser].name = network_analyst.model[
recloser].name.replace('goab', 'recloser')
network_analyst.model[recloser].nameclass = 'recloser'
network_analyst.model.set_names()
#Compute the metrics if needed
if compute_metrics:
#Compute metrics
network_analyst.compute_all_metrics_per_feeder(
compute_kva_density_with_transformers=
compute_kva_density_with_transformers)
#Export metrics to Excel
network_analyst.export(excel_output)
#Export metrics to JSON
network_analyst.export_json(json_output)
#Return the model
return network_analyst.model
def _topological_sort(self) -> Iterable[Step]:
return self._steps_iterable(nx.topological_sort(self))
def build_recipes(recipe_folder: str, config_path: str, recipes: List[str],
mulled_test: bool = True, testonly: bool = False,
force: bool = False,
docker_builder: docker_utils.RecipeBuilder = None,
label: str = None,
anaconda_upload: bool = False,
mulled_upload_target=None,
check_channels: List[str] = None,
do_lint: bool = None,
lint_exclude: List[str] = None,
n_workers: int = 1,
worker_offset: int = 0,
keep_old_work: bool = False):
"""
Build one or many bioconda packages.
Arguments:
recipe_folder: Directory containing possibly many, and possibly nested, recipes.
config_path: Path to config file
packages: Glob indicating which packages should be considered. Note that packages
matching the glob will still be filtered out by any blacklists
specified in the config.
mulled_test: If true, test the package in a minimal container.
testonly: If true, only run test.
force: If true, build the recipe even though it would otherwise be filtered out.
docker_builder: If specified, use to build all recipes
label: If specified, use to label uploaded packages on anaconda. Default is "main" label.
anaconda_upload: If true, upload the package(s) to anaconda.org.
mulled_upload_target: If specified, upload the mulled docker image to the given target
on quay.io.
check_channels: Channels to check to see if packages already exist in them.
Defaults to every channel in the config file except "defaults".
do_lint: Whether to run linter
lint_exclude: List of linting functions to exclude.
n_workers: The number of parallel instances of bioconda-utils being run. The
sub-DAGs are then split into groups of n_workers size.
worker_offset: If n_workers is >1, then every worker_offset within a given group of
sub-DAGs will be processed.
keep_old_work: Do not remove anything from environment, even after successful build and test.
"""
if not recipes:
logger.info("Nothing to be done.")
return True
config = utils.load_config(config_path)
blacklist = utils.get_blacklist(config, recipe_folder)
# get channels to check
if check_channels is None:
if config['channels']:
check_channels = [c for c in config['channels'] if c != "defaults"]
else:
check_channels = []
# setup linting
if do_lint:
always_exclude = ('build_number_needs_bump',)
if not lint_exclude:
lint_exclude = always_exclude
else:
lint_exclude = tuple(set(lint_exclude) | set(always_exclude))
linter = lint.Linter(config, recipe_folder, lint_exclude)
else:
linter = None
failed = []
dag, name2recipes = graph.build(recipes, config=config_path, blacklist=blacklist)
if not dag:
logger.info("Nothing to be done.")
return True
skip_dependent = defaultdict(list)
dag = remove_cycles(dag, name2recipes, failed, skip_dependent)
subdag = get_subdags(dag, n_workers, worker_offset)
if not subdag:
logger.info("Nothing to be done.")
return True
logger.info("%i recipes to build and test: \n%s", len(subdag), "\n".join(subdag.nodes()))
recipe2name = {}
for name, recipe_list in name2recipes.items():
for recipe in recipe_list:
recipe2name[recipe] = name
recipes = [(recipe, recipe2name[recipe])
for package in nx.topological_sort(subdag)
for recipe in name2recipes[package]]
built_recipes = []
skipped_recipes = []
failed_uploads = []
for recipe, name in recipes:
if name in skip_dependent:
logger.info('BUILD SKIP: skipping %s because it depends on %s '
'which had a failed build.',
recipe, skip_dependent[name])
skipped_recipes.append(recipe)
continue
logger.info('Determining expected packages for %s', recipe)
try:
pkg_paths = utils.get_package_paths(recipe, check_channels, force=force)
except utils.DivergentBuildsError as exc:
logger.error('BUILD ERROR: packages with divergent build strings in repository '
'for recipe %s. A build number bump is likely needed: %s',
recipe, exc)
failed.append(recipe)
for pkg in nx.algorithms.descendants(subdag, name):
skip_dependent[pkg].append(recipe)
continue
except UnsatisfiableError as exc:
logger.error('BUILD ERROR: could not determine dependencies for recipe %s: %s',
recipe, exc)
failed.append(recipe)
for pkg in nx.algorithms.descendants(subdag, name):
skip_dependent[pkg].append(recipe)
continue
if not pkg_paths:
logger.info("Nothing to be done for recipe %s", recipe)
continue
# If a recipe depends on conda, it means it must be installed in
# the root env, which is not compatible with mulled-build tests. In
# that case, we temporarily disable the mulled-build tests for the
# recipe.
keep_mulled_test = not has_conda_in_build(recipe)
if mulled_test and not keep_mulled_test:
logger.info('TEST SKIP: skipping mulled-build test for %s because it '
'depends on conda or conda-build', recipe)
res = build(recipe=recipe,
pkg_paths=pkg_paths,
testonly=testonly,
mulled_test=mulled_test and keep_mulled_test,
channels=config['channels'],
docker_builder=docker_builder,
linter=linter)
if not res.success:
failed.append(recipe)
for pkg in nx.algorithms.descendants(subdag, name):
skip_dependent[pkg].append(recipe)
else:
built_recipes.append(recipe)
if not testonly:
if anaconda_upload:
for pkg in pkg_paths:
if not upload.anaconda_upload(pkg, label=label):
failed_uploads.append(pkg)
if mulled_upload_target and keep_mulled_test:
for img in res.mulled_images:
upload.mulled_upload(img, mulled_upload_target)
# remove traces of the build
if not keep_old_work:
conda_build_purge()
if failed or failed_uploads:
logger.error('BUILD SUMMARY: of %s recipes, '
'%s failed and %s were skipped. '
'Details of recipes and environments follow.',
len(recipes), len(failed), len(skipped_recipes))
if built_recipes:
logger.error('BUILD SUMMARY: while the entire build failed, '
'the following recipes were built successfully:\n%s',
'\n'.join(built_recipes))
for recipe in failed:
logger.error('BUILD SUMMARY: FAILED recipe %s', recipe)
for name, dep in skip_dependent.items():
logger.error('BUILD SUMMARY: SKIPPED recipe %s '
'due to failed dependencies %s', name, dep)
if failed_uploads:
logger.error('UPLOAD SUMMARY: the following packages failed to upload:\n%s',
'\n'.join(failed_uploads))
return False
logger.info("BUILD SUMMARY: successfully built %s of %s recipes",
len(built_recipes), len(recipes))
return True
def make_sorted_analysis_unit_dict_from_function_graph(
self, analysis_unit_dict):
''' BUILDS A TOPO SORTED FUNCTION GRAPH. THIS ALLOWS THE ANALYSIS TO START ON FUNCTION LEAFS,
SO WE CAN HOPEFULLY DISCOVER UNITS ON THE RETURN TYPE AND PROPAGE THEM UP. THE
FUNCTION GRAPH MAY HAVE CYCLES (recursion, for example), THEREFORE WE REMOVE THESE EDGES FROM THE GRAPH
AND ANALYZE THEM LAST (<-- not sure this is best)
input: a dictionary of functions from this dump file
output: OrderedDict of functions
postcondition: returned dict must be the same length as the input dict, and contain all the same elements
'''
if self.debug_verbose:
print inspect.stack()[0][3]
return_dict = OrderedDict()
G = self.function_graph
# TRY FINDING A DAG. IF NOT, REMOVE EDGES AND TRY AGAIN.
super_break = 0
while nx.number_of_nodes(G) > 0 and super_break < 1000:
super_break += 1
if not nx.is_directed_acyclic_graph(G):
try:
# SEARCH FOR CYCLE AND REMOVE EDGES
edges = nx.find_cycle(G)
G.remove_edges_from(edges)
if self.debug and self.debug_print_function_graph:
print 'Function graph has cycle %s' % edges,
except:
if self.debug and self.debug_print_function_graph:
print 'Function graph is not a DAG and does not have a cycle!'
# GIVE UP AND RETURN UNSORTED
return analysis_unit_dict
else:
# WE HAVE A DIGRAPH, CAN PROCEED ( and topo sort )
break
if nx.number_of_nodes(G) == 0:
# RETURN UNCHANGED
return analysis_unit_dict
# WE HAVE A DIRECTED GRAPH WITH NODES, CAN SORT AND ADD NODES TO ORDERED LIST
function_graph_topo_sort = nx.topological_sort(G)
function_graph_topo_sort_reversed = function_graph_topo_sort[::
-1] # REVERSED
# OPTIONAL DEBUG PRINT
if self.debug_print_function_topo_sort:
print function_graph_topo_sort_reversed
# CREATE RETURN DICT FROM TOPO SORT
for node in function_graph_topo_sort_reversed:
function_id_attr_dict = nx.get_node_attributes(G, 'function_id')
#print function_id_attr_dict
if node in function_id_attr_dict: # FIRST OPTION SHOULD SHORTCUT WHEN ATTR DICT DOES NOT EXITS
# ADD FUNCTION TO NEW DICTIONARY - THIS IS THE EXPLORE ORDER
return_dict[function_id_attr_dict[node]] = analysis_unit_dict[
function_id_attr_dict[node]]
else:
if self.debug and self.debug_print_function_graph:
print "Graph node does not have function_dict_key"
# ADD ANY REMAINING FUNCTIONS NOT IN THE TOPO SORT TO THE ORDERED DICT
for k in analysis_unit_dict.keys():
if k not in return_dict:
return_dict[k] = analysis_unit_dict[k]
assert (len(return_dict) == len(analysis_unit_dict))
return return_dict
def build_graph(graph):
"""
Instantiate Modules used in graph and fill the ModuleTable.
Args:
graph: A instance of class:ModuleGraph.
Returns:
ModuleTable: it holds all information of modules(nodes) and relations(edges) between them.
toposort: Topological sorting of the graph, it's a list-like object
Raises:
Stop: Or one of its subclasses if an error occurs while building the graph.
TypeError: If modules are of an inappropriate type.
"""
# instantiation
nodes = graph.nodes
modules = []
for node_name in nodes:
module_name = graph.node[node_name]['attr']['module']
p = __import__(module_name, globals(), locals(), level=0)
globals()[module_name] = p.__dict__[module_name]
modules.append(eval(module_name + '()'))
# print(eval(module_name).make_module_description().input_names())
# check type
def _Check_Module(md):
"""
Check Modules which will be registered to ModuleTable.
Args:
md (Module or [Module]): a Module or a list of Modules
Returns:
succeed or not
"""
if isinstance(md, (list, tuple)):
for x in md:
_Check_Module(x)
return
# assert isinstance(md, Module), md
# assert not isinstance(modules, Modules), \
# "Cannot append more Modules after BaseSession was setup!"
return True
# Check_Module = _Check_Module
#
# for md in modules:
# Check_Module(md)
# register to ModuleTable
logger.logger.info("Build graph ...")
ModuleTable = {
'node_name': {
'module_name': None,
'module_instance': None,
'module_desc': None,
'output': None
}
}
for index, node_name in enumerate(nodes):
logger.logger.info('Register ' + node_name + ' to session')
ModuleTable[node_name] = {
'module_name': graph.node[node_name]['attr']['module'],
'module_instance': modules[index],
'module_desc': modules[index].make_module_description(),
'output': None
}
toposort = list(nx.topological_sort(graph))
return ModuleTable, toposort
def quasi_topological_sort_nodes(graph, nodes=None):
"""
Sort a given set of nodes from a graph based on the following rules:
# - if A -> B and not B -> A, then we have A < B
# - if A -> B and B -> A, then the ordering is undefined
Following the above rules gives us a quasi-topological sorting of nodes in the graph. It also works for cyclic
graphs.
:param networkx.DiGraph graph: A local transition graph of the function.
:param iterable nodes: A list of nodes to sort. None if you want to sort all nodes inside the graph.
:return: A list of ordered nodes.
:rtype: list
"""
# fast path for single node graphs
if graph.number_of_nodes() == 1:
return graph.nodes()
# make a copy to the graph since we are gonna modify it
graph_copy = networkx.DiGraph()
# find all strongly connected components in the graph
sccs = [
scc for scc in networkx.strongly_connected_components(graph)
if len(scc) > 1
]
# collapse all strongly connected components
for src, dst in graph.edges():
scc_index = CFGUtils._components_index_node(sccs, src)
if scc_index is not None:
src = SCCPlaceholder(scc_index)
scc_index = CFGUtils._components_index_node(sccs, dst)
if scc_index is not None:
dst = SCCPlaceholder(scc_index)
if isinstance(src, SCCPlaceholder) and isinstance(
dst, SCCPlaceholder) and src == dst:
continue
if src == dst:
continue
graph_copy.add_edge(src, dst)
# add loners
out_degree_zero_nodes = [
node for (node, degree) in graph.out_degree() if degree == 0
]
for node in out_degree_zero_nodes:
if graph.in_degree(node) == 0:
graph_copy.add_node(node)
# topological sort on acyclic graph `graph_copy`
tmp_nodes = networkx.topological_sort(graph_copy)
ordered_nodes = []
for n in tmp_nodes:
if isinstance(n, SCCPlaceholder):
CFGUtils._append_scc(graph, ordered_nodes, sccs[n.scc_id])
else:
ordered_nodes.append(n)
if nodes is None:
return ordered_nodes
nodes = set(nodes)
ordered_nodes = [n for n in ordered_nodes if n in nodes]
return ordered_nodes
def install_cmd(ctx, force=False, skip_tests=False, pkg_json=None):
if not pkg_json:
project = ctx.obj['PROJECT']
if not project.root:
echo("Not in a package project directory.")
ctx.abort()
if str(Path(os.getcwd()).parent) != project.root:
echo("Not in a package directory.")
ctx.abort()
if not os.path.isfile('pkg.json'):
echo(
"Not in a package directory, 'pkg.json' not found in the current direcotry."
)
ctx.abort()
install_dest = project.root
pkg_json = os.path.join(os.getcwd(), 'pkg.json')
else:
install_dest = os.path.dirname(
os.path.dirname(pkg_json)) # parent dir of where pkg.json is
try:
package = Package(pkg_json=pkg_json)
dep_graph = nx.DiGraph()
build_dep_graph(package, DG=dep_graph)
except Exception as ex:
echo(f"Unable to build package dependency graph: {ex}")
ctx.abort()
# exclude the first one which is the current package itself, it's important to reverse the order
dep_pkgs = list(reversed(list(nx.topological_sort(dep_graph))[1:]))
if dep_pkgs:
echo("Start dependency installation.")
else:
echo("No dependency defined, no installation needed.")
failed_pkgs = []
installed_pkgs = []
for dep_pkg_uri in dep_pkgs:
package = Package(pkg_uri=dep_pkg_uri)
installed = False
try:
path = package.install(install_dest, force=force)
installed = True
installed_pkgs.append(dep_pkg_uri)
echo(
f"Package installed in: {path.replace(os.path.join(os.getcwd(), ''), '')}"
)
except Exception as ex:
echo(f"{ex}")
failed_pkgs.append(dep_pkg_uri)
if not skip_tests and installed:
test_package(path)
return installed_pkgs, failed_pkgs
if scene[1] == 'None': #parallel of previous == None - get first scene
G.add_node(scene[0])
labels.append(scene[0])
target = scene[0]
else:
pass
def chain_scenes(scenes, target):
for scene in scenes:
if scene[1] == target:
G.add_edge(target, scene[0])
labels.append(scene[0])
target = scene[0]
chain_scenes(scenes, target)
else:
pass
chain_scenes(scenes, target)
srt = nx.topological_sort(G)
print(srt)
pos = nx.spectral_layout(G)
nx.draw_networkx_nodes(G, pos)
nx.draw_networkx_labels(G, pos)
nx.draw_networkx_edges(G, pos)
plt.show()
def _get_topological_order(self):
return list(nx.topological_sort(self.graph))
def topological_sort(self):
"""Return a list of nodes in this graph in topological sort order."""
return nx.topological_sort(self)
def solve(num_wizards, num_constraints, wizards, constraints):
"""
Write your algorithm here.
Input:
num_wizards: Number of wizards
num_constraints: Number of constraints
wizards: An array of wizard names, in no particular order
constraints: A 2D-array of constraints,
where constraints[0] may take the form ['A', 'B', 'C']i
Output:
An array of wizard names in the ordering your algorithm returns
"""
global wiz_const
wiz_const = mapConstraints(wizards, constraints)
partial_soltns = []
# counter for priority queue since it doesn't allow
# identical priorities
k = 0
# list of wizards sorted by lowest to highest degree
sorted_wiz = sortWizByConsts(wiz_const)
wiz_rankings = {wiz: i for i, wiz in enumerate(sorted_wiz)}
const_set = set(map(tuple, constraints))
for i in range(4):
heapq.heappush(partial_soltns, (0, k, nx.DiGraph(), const_set.copy()))
k += 1
print("setup done, commencing solving")
while len(partial_soltns):
# for partial_soltn, const_set in partial_soltns :
# partial_soltns.remove(partial_soltn)
num_seen, _, partial_soltn, const_set = heapq.heappop(partial_soltns)
const = findNextConst(partial_soltn, const_set, wiz_rankings)
print("seen " + str(len(partial_soltn)) +
"\t num partial_solutions\t" + str(len(partial_soltns)))
try:
const_set.remove(const)
except KeyError:
print("BAD SHIT")
pass
possible_arrangements = [(const[0], const[1], const[2]),
(const[2], const[0], const[1]),
(const[2], const[1], const[0]),
(const[1], const[0], const[2])]
for arr in possible_arrangements:
soltn = partial_soltn.copy()
a, b, c = arr
if not (soltn.has_node(a) and soltn.has_node(b)
and nx.has_path(soltn, a, b)):
soltn.add_edge(a, b)
if not (soltn.has_node(b) and soltn.has_node(c)
and nx.has_path(soltn, b, c)):
soltn.add_edge(b, c)
# see if we violated any other constraints (seen or not seen)
is_valid, num_wiz = validNumWiz(soltn, const_set)
if is_valid and len(list(nx.simple_cycles(soltn))) == 0:
heapq.heappush(partial_soltns,
(-len(soltn), k, soltn, const_set.copy()))
k += 1
# are we done?
if num_wiz == num_wizards:
print(
"FINAL SOLUTION (found without processing all constraints but validating against them)"
)
ordering = list(nx.topological_sort(soltn))
finishEverything(ordering, constraints)
return ordering
if foundCompleteOrdering(heapq.heappop(partial_soltns)):
print("FINAL SOLUTION")
ordering = list(nx.topological_sort(soltn))
finishEverything(ordering, constraints)
return ordering
print("NO SOLUTION FOUND")
return ""
def _write_detailed_dot(graph, dotfilename):
"""Create a dot file with connection info
digraph structs {
node [shape=record];
struct1 [label="<f0> left|<f1> mid\ dle|<f2> right"];
struct2 [label="<f0> one|<f1> two"];
struct3 [label="hello\nworld |{ b |{c|<here> d|e}| f}| g | h"];
struct1:f1 -> struct2:f0;
struct1:f0 -> struct2:f1;
struct1:f2 -> struct3:here;
}
"""
text = ['digraph structs {', 'node [shape=record];']
# write nodes
edges = []
replacefunk = lambda x: x.replace('_', '').replace('.', ''). \
replace('@', '').replace('-', '')
for n in nx.topological_sort(graph):
nodename = str(n)
inports = []
for u, v, d in graph.in_edges_iter(nbunch=n, data=True):
for cd in d['connect']:
if isinstance(cd[0], string_types):
outport = cd[0]
else:
outport = cd[0][0]
inport = cd[1]
ipstrip = 'in' + replacefunk(inport)
opstrip = 'out' + replacefunk(outport)
edges.append('%s:%s:e -> %s:%s:w;' % (str(u).replace('.', ''),
opstrip,
str(v).replace('.', ''),
ipstrip))
if inport not in inports:
inports.append(inport)
inputstr = '{IN'
for ip in sorted(inports):
inputstr += '|<in%s> %s' % (replacefunk(ip), ip)
inputstr += '}'
outports = []
for u, v, d in graph.out_edges_iter(nbunch=n, data=True):
for cd in d['connect']:
if isinstance(cd[0], string_types):
outport = cd[0]
else:
outport = cd[0][0]
if outport not in outports:
outports.append(outport)
outputstr = '{OUT'
for op in sorted(outports):
outputstr += '|<out%s> %s' % (replacefunk(op), op)
outputstr += '}'
srcpackage = ''
if hasattr(n, '_interface'):
pkglist = n._interface.__class__.__module__.split('.')
if len(pkglist) > 2:
srcpackage = pkglist[2]
srchierarchy = '.'.join(nodename.split('.')[1:-1])
nodenamestr = '{ %s | %s | %s }' % (nodename.split('.')[-1],
srcpackage,
srchierarchy)
text += ['%s [label="%s|%s|%s"];' % (nodename.replace('.', ''),
inputstr,
nodenamestr,
outputstr)]
# write edges
for edge in sorted(edges):
text.append(edge)
text.append('}')
filep = open(dotfilename, 'wt')
filep.write('\n'.join(text))
filep.close()
return text
def traverse(self):
order = list(nx.topological_sort(self.graph))
for o in order:
n = self.graph.nodes[o]
self.data = n['data'].execute(self.data)
def _get_dot(self,
prefix=None,
hierarchy=None,
colored=False,
simple_form=True,
level=0):
"""Create a dot file with connection info
"""
import networkx as nx
if prefix is None:
prefix = ' '
if hierarchy is None:
hierarchy = []
colorset = [
'#FFFFC8', # Y
'#0000FF',
'#B4B4FF',
'#E6E6FF', # B
'#FF0000',
'#FFB4B4',
'#FFE6E6', # R
'#00A300',
'#B4FFB4',
'#E6FFE6', # G
'#0000FF',
'#B4B4FF'
] # loop B
if level > len(colorset) - 2:
level = 3 # Loop back to blue
dotlist = ['%slabel="%s";' % (prefix, self.name)]
for node in nx.topological_sort(self._graph):
fullname = '.'.join(hierarchy + [node.fullname])
nodename = fullname.replace('.', '_')
if not isinstance(node, Workflow):
node_class_name = get_print_name(node, simple_form=simple_form)
if not simple_form:
node_class_name = '.'.join(node_class_name.split('.')[1:])
if hasattr(node, 'iterables') and node.iterables:
dotlist.append(('%s[label="%s", shape=box3d,'
'style=filled, color=black, colorscheme'
'=greys7 fillcolor=2];') %
(nodename, node_class_name))
else:
if colored:
dotlist.append(
('%s[label="%s", style=filled,'
' fillcolor="%s"];') %
(nodename, node_class_name, colorset[level]))
else:
dotlist.append(
('%s[label="%s"];') % (nodename, node_class_name))
for node in nx.topological_sort(self._graph):
if isinstance(node, Workflow):
fullname = '.'.join(hierarchy + [node.fullname])
nodename = fullname.replace('.', '_')
dotlist.append('subgraph cluster_%s {' % nodename)
if colored:
dotlist.append(prefix + prefix + 'edge [color="%s"];' %
(colorset[level + 1]))
dotlist.append(prefix + prefix + 'style=filled;')
dotlist.append(prefix + prefix + 'fillcolor="%s";' %
(colorset[level + 2]))
dotlist.append(
node._get_dot(prefix=prefix + prefix,
hierarchy=hierarchy + [self.name],
colored=colored,
simple_form=simple_form,
level=level + 3))
dotlist.append('}')
else:
for subnode in self._graph.successors(node):
if node._hierarchy != subnode._hierarchy:
continue
if not isinstance(subnode, Workflow):
nodefullname = '.'.join(hierarchy + [node.fullname])
subnodefullname = '.'.join(hierarchy +
[subnode.fullname])
nodename = nodefullname.replace('.', '_')
subnodename = subnodefullname.replace('.', '_')
for _ in self._graph.get_edge_data(node,
subnode)['connect']:
dotlist.append('%s -> %s;' %
(nodename, subnodename))
logger.debug('connection: %s', dotlist[-1])
# add between workflow connections
for u, v, d in self._graph.edges(data=True):
uname = '.'.join(hierarchy + [u.fullname])
vname = '.'.join(hierarchy + [v.fullname])
for src, dest in d['connect']:
uname1 = uname
vname1 = vname
if isinstance(src, tuple):
srcname = src[0]
else:
srcname = src
if '.' in srcname:
uname1 += '.' + '.'.join(srcname.split('.')[:-1])
if '.' in dest and '@' not in dest:
if not isinstance(v, Workflow):
if 'datasink' not in \
str(v._interface.__class__).lower():
vname1 += '.' + '.'.join(dest.split('.')[:-1])
else:
vname1 += '.' + '.'.join(dest.split('.')[:-1])
if uname1.split('.')[:-1] != vname1.split('.')[:-1]:
dotlist.append(
'%s -> %s;' %
(uname1.replace('.', '_'), vname1.replace('.', '_')))
logger.debug('cross connection: %s', dotlist[-1])
return ('\n' + prefix).join(dotlist)
def compute(self):
def getName(pref, id):
while '%s%d' % (pref, id) in self.depTree:
id += 1
return '%s%d' % (pref, id)
self.depTree = self.inGraph.copy()
self.transitveReduction(self.depTree)
tsort = list(reversed(list(NX.topological_sort(self.depTree))))
nextId = 0
for node in tsort:
#strip out cycles within parents
parents = set([i[0] for i in self.depTree.in_edges(node)])
if len(parents) > 1:
for parent in parents:
for pedge in self.depTree.out_edges(parent):
if pedge[1] in parents:
self.depTree.remove_edge(parent, node)
break
parents = [i[0] for i in self.depTree.in_edges(node)]
if len(parents) > 1:
# insert new "virtual" node as parent to all parents
vpNode = getName("Virtual", nextId)
self.depTree.add_node(vpNode, virtual='1')
# insert new "virtual follow-on" node as follow up to above node
vfoNode = getName("VirtualF", nextId)
self.depTree.add_node(vfoNode, virtual='1')
self.depTree.add_edge(vpNode, vfoNode, followOn='1')
nextId += 1
for parent in parents:
# insert vpNode above all the parents
for inEdge in self.depTree.in_edges(parent):
if not self.isFollowOn(inEdge[0], inEdge[1]):
self.depTree.add_edge(inEdge[0], vpNode)
self.depTree.remove_edge(inEdge[0], inEdge[1])
# add parent as child of the followOn
self.depTree.add_edge(vfoNode, parent)
# remove node from parent
self.depTree.remove_edge(parent, node)
# add parent's childrent to vpNode
for child in list(self.depTree.successors(parent)):
if not self.isFollowOn(parent, child):
self.depTree.remove_edge(parent, child)
self.depTree.add_edge(vpNode, child)
if self.depTree.has_edge(child, vpNode):
self.depTree.remove_edge(child, vpNode)
#add node to vpNode
self.depTree.add_edge(vpNode, node)
#lazy clean-up of inserted transitive edges
#will need to not do this brute-force for bigger trees
self.transitveReduction(self.depTree)
assert (len(self.depTree.in_edges(node)) == 1)
# process virtual node next
tsort.insert(tsort.index(node) + 1, vpNode)
assert len(self.depTree.in_edges(node)) < 2
for node in tsort:
assert len(self.depTree.in_edges(node)) < 2
assert NX.is_directed_acyclic_graph(self.depTree)
self.enforceMaxParallel()
import networkx as nx
import pygraphviz as pgv
from nxpd import draw, nxpdParams
nxpdParams['show'] = 'ipynb'
G = nx.DiGraph()
G.add_edges_from([('a', 'b'), ('b', 'c'), ('b', 'd'), ('d', 'c'), ('a', 'd')])
draw(G, layout='circo')
# if the directed graph is acyclic then it has a topological ordering
if nx.is_directed_acyclic_graph(G):
print("Topological ordering of the nodes:", nx.topological_sort(G))
else:
print("G contains a cycle, hence it cannot be topologically sorted.")
def ordered(self):
return iter(topological_sort(self.graph))
def build_recipes(
recipe_folder,
config,
packages="*",
mulled_test=True,
testonly=False,
force=False,
docker_builder=None,
label=None,
anaconda_upload=False,
mulled_upload_target=None,
check_channels=None,
lint_args=None,
):
"""
Build one or many bioconda packages.
Parameters
----------
recipe_folder : str
Directory containing possibly many, and possibly nested, recipes.
config : str or dict
If string, path to config file; if dict then assume it's an
already-parsed config file.
packages : str
Glob indicating which packages should be considered. Note that packages
matching the glob will still be filtered out by any blacklists
specified in the config.
mulled_test : bool
If True, then test the package in a minimal container.
testonly : bool
If True, only run test.
force : bool
If True, build the recipe even though it would otherwise be filtered
out.
docker_builder : docker_utils.RecipeBuilder instance
If not None, then use this RecipeBuilder to build all recipes.
label : str
Optional label to use when uploading packages. Useful for testing and
debugging. Default is to use the "main" label.
anaconda_upload : bool
If True, upload the package to anaconda.org.
mulled_upload_target : None
If not None, upload the mulled docker image to the given target on quay.io.
check_channels : list
Channels to check to see if packages already exist in them. If None,
then defaults to every channel in the config file except "defaults".
lint_args : linting.LintArgs | None
If not None, then apply linting just before building.
"""
orig_config = config
config = utils.load_config(config)
blacklist = utils.get_blacklist(config['blacklists'], recipe_folder)
if check_channels is None:
if config['channels']:
check_channels = [c for c in config['channels'] if c != "defaults"]
else:
check_channels = []
logger.info('blacklist: %s', ', '.join(sorted(blacklist)))
if packages == "*":
packages = ["*"]
recipes = []
for package in packages:
for recipe in utils.get_recipes(recipe_folder, package):
if os.path.relpath(recipe, recipe_folder) in blacklist:
logger.debug('blacklisted: %s', recipe)
continue
recipes.append(recipe)
logger.debug(recipe)
if not recipes:
logger.info("Nothing to be done.")
return True
logger.debug('recipes: %s', recipes)
if lint_args is not None:
lint_exclude = (lint_args.exclude or ())
if 'already_in_bioconda' not in lint_exclude:
lint_exclude = tuple(lint_exclude) + ('already_in_bioconda', )
lint_args = linting.LintArgs(lint_exclude, lint_args.registry)
dag, name2recipes = graph.build(recipes,
config=orig_config,
blacklist=blacklist)
recipe2name = {}
for k, v in name2recipes.items():
for i in v:
recipe2name[i] = k
if not dag:
logger.info("Nothing to be done.")
return True
else:
logger.info("Building and testing %s recipes in total", len(dag))
logger.info("Recipes to build: \n%s", "\n".join(dag.nodes()))
subdags_n = int(os.environ.get("SUBDAGS", 1))
subdag_i = int(os.environ.get("SUBDAG", 0))
if subdag_i >= subdags_n:
raise ValueError("SUBDAG=%s (zero-based) but only SUBDAGS=%s "
"subdags are available")
failed = []
skip_dependent = defaultdict(list)
# Get connected subdags and sort by nodes
if testonly:
# use each node as a subdag (they are grouped into equal sizes below)
subdags = sorted([[n] for n in nx.nodes(dag)])
else:
# take connected components as subdags, remove cycles
subdags = []
for cc_nodes in nx.connected_components(dag.to_undirected()):
cc = dag.subgraph(sorted(cc_nodes))
nodes_in_cycles = set()
for cycle in list(nx.simple_cycles(cc)):
logger.error(
'BUILD ERROR: '
'dependency cycle found: %s',
cycle,
)
nodes_in_cycles.update(cycle)
for name in sorted(nodes_in_cycles):
cycle_fail_recipes = sorted(name2recipes[name])
logger.error(
'BUILD ERROR: '
'cannot build recipes for %s since it cyclically depends '
'on other packages in the current build job. Failed '
'recipes: %s',
name,
cycle_fail_recipes,
)
failed.extend(cycle_fail_recipes)
for n in nx.algorithms.descendants(cc, name):
if n in nodes_in_cycles:
continue # don't count packages twice (failed/skipped)
skip_dependent[n].extend(cycle_fail_recipes)
cc_without_cycles = dag.subgraph(name for name in cc
if name not in nodes_in_cycles)
# ensure that packages which need a build are built in the right order
subdags.append(nx.topological_sort(cc_without_cycles))
# chunk subdags such that we have at most subdags_n many
if subdags_n < len(subdags):
chunks = [[n for subdag in subdags[i::subdags_n] for n in subdag]
for i in range(subdags_n)]
else:
chunks = subdags
if subdag_i >= len(chunks):
logger.info("Nothing to be done.")
return True
# merge subdags of the selected chunk
subdag = dag.subgraph(chunks[subdag_i])
recipes = [
recipe for package in subdag for recipe in name2recipes[package]
]
logger.info("Building and testing subdag %s of %s (%s recipes)",
subdag_i + 1, subdags_n, len(recipes))
built_recipes = []
skipped_recipes = []
all_success = True
failed_uploads = []
for recipe in recipes:
recipe_success = True
name = recipe2name[recipe]
if name in skip_dependent:
logger.info(
'BUILD SKIP: '
'skipping %s because it depends on %s '
'which had a failed build.', recipe, skip_dependent[name])
skipped_recipes.append(recipe)
continue
logger.info('Determining expected packages')
try:
pkg_paths = utils.get_package_paths(recipe,
check_channels,
force=force)
except utils.DivergentBuildsError as e:
logger.error(
'BUILD ERROR: '
'packages with divergent build strings in repository '
'for recipe %s. A build number bump is likely needed: %s',
recipe, e)
failed.append(recipe)
for n in nx.algorithms.descendants(subdag, name):
skip_dependent[n].append(recipe)
continue
except UnsatisfiableError as e:
logger.error(
'BUILD ERROR: '
'could not determine dependencies for recipe %s: %s', recipe,
e)
failed.append(recipe)
for n in nx.algorithms.descendants(subdag, name):
skip_dependent[n].append(recipe)
continue
if not pkg_paths:
logger.info("Nothing to be done for recipe %s", recipe)
continue
# If a recipe depends on conda, it means it must be installed in
# the root env, which is not compatible with mulled-build tests. In
# that case, we temporarily disable the mulled-build tests for the
# recipe.
deps = []
deps += utils.get_deps(recipe, orig_config, build=True)
deps += utils.get_deps(recipe, orig_config, build=False)
keep_mulled_test = True
if 'conda' in deps or 'conda-build' in deps:
keep_mulled_test = False
if mulled_test:
logger.info(
'TEST SKIP: '
'skipping mulled-build test for %s because it '
'depends on conda or conda-build', recipe)
res = build(
recipe=recipe,
recipe_folder=recipe_folder,
pkg_paths=pkg_paths,
testonly=testonly,
mulled_test=mulled_test and keep_mulled_test,
force=force,
channels=config['channels'],
docker_builder=docker_builder,
lint_args=lint_args,
)
all_success &= res.success
recipe_success &= res.success
if not res.success:
failed.append(recipe)
for n in nx.algorithms.descendants(subdag, name):
skip_dependent[n].append(recipe)
elif not testonly:
for pkg in pkg_paths:
# upload build
if anaconda_upload:
if not upload.anaconda_upload(pkg, label=label):
failed_uploads.append(pkg)
if mulled_upload_target and keep_mulled_test:
for img in res.mulled_images:
upload.mulled_upload(img, mulled_upload_target)
# remove traces of the build
purge()
if recipe_success:
built_recipes.append(recipe)
if failed or failed_uploads:
logger.error(
'BUILD SUMMARY: of %s recipes, '
'%s failed and %s were skipped. '
'Details of recipes and environments follow.', len(recipes),
len(failed), len(skipped_recipes))
if len(built_recipes) > 0:
logger.error(
'BUILD SUMMARY: while the entire build failed, '
'the following recipes were built successfully:\n%s',
'\n'.join(built_recipes))
for recipe in failed:
logger.error('BUILD SUMMARY: FAILED recipe %s', recipe)
for name, dep in skip_dependent.items():
logger.error(
'BUILD SUMMARY: SKIPPED recipe %s '
'due to failed dependencies %s', name, dep)
if failed_uploads:
logger.error(
'UPLOAD SUMMARY: the following packages failed to upload:\n%s',
'\n'.join(failed_uploads))
return False
logger.info("BUILD SUMMARY: successfully built %s of %s recipes",
len(built_recipes), len(recipes))
return all_success
def initialize(self):
"""
Initialize Graph class instance.
Initialization includes: create NetworkX DiGraph,
populate it with input and step nodes, and directed edges.
Args:
None.
Returns:
On failure: Raises WorkflowDAGException.
"""
for context in self._exec_contexts | self._data_contexts:
# set default empty values for context options
if context not in self._context_options:
self._context_options[context] = {}
# references to step classes for each context
try:
self._load_context_classes()
except WorkflowDAGException as err:
msg = 'cannot load context-specific step classes'
Log.an().error(msg)
raise WorkflowDAGException(str(err)+'|'+msg) from err
# flatten parameters
self._parameters = {
param_name: param['value']
for param_name, param in self._workflow['parameters'].items()
}
# init DAG object with structure and empty nodes
self._graph = nx.DiGraph()
try:
self._init_graph_structure()
except WorkflowDAGException as err:
msg = 'cannot initialize graph structure'
Log.an().error(msg)
raise WorkflowDAGException(str(err)+'|'+msg) from err
# validate that graph is a DAG
if not nx.is_directed_acyclic_graph(self._graph):
msg = 'graph contains cycles, check step dependencies'
Log.an().error(msg)
raise WorkflowDAGException(msg)
# topological sort of graph nodes
self._topo_sort = list(nx.topological_sort(self._graph))
# create URIs for each input and step for all contexts
try:
self._init_context_uris()
except WorkflowDAGException as err:
msg = 'cannot initialize context uris'
Log.an().error(msg)
raise WorkflowDAGException(str(err)+'|'+msg) from err
# initalize input nodes
try:
self._init_inputs()
except WorkflowDAGException as err:
msg = 'cannot initialize workflow inputs'
Log.an().error(msg)
raise WorkflowDAGException(str(err)+'|'+msg) from err
# initialize step nodes
try:
self._init_steps()
except WorkflowDAGException as err:
msg = 'cannot initialize workflow steps'
Log.an().error(msg)
raise WorkflowDAGException(str(err)+'|'+msg) from err
def export(self,
filename=None,
prefix="output",
format="python",
include_config=False):
"""Export object into a different format
Parameters
----------
filename: string
file to save the code to; overrides prefix
prefix: string
prefix to use for output file
format: string
one of "python"
include_config: boolean
whether to include node and workflow config values
"""
import networkx as nx
formats = ["python"]
if format not in formats:
raise ValueError('format must be one of: %s' % '|'.join(formats))
flatgraph = self._create_flat_graph()
nodes = nx.topological_sort(flatgraph)
all_lines = None
lines = ['# Workflow']
importlines = [
'from nipype.pipeline.engine import Workflow, '
'Node, MapNode'
]
functions = {}
if format == "python":
connect_template = '%s.connect(%%s, %%s, %%s, "%%s")' % self.name
connect_template2 = '%s.connect(%%s, "%%s", %%s, "%%s")' \
% self.name
wfdef = '%s = Workflow("%s")' % (self.name, self.name)
lines.append(wfdef)
if include_config:
lines.append('%s.config = %s' % (self.name, self.config))
for idx, node in enumerate(nodes):
nodename = node.fullname.replace('.', '_')
# write nodes
nodelines = format_node(node,
format='python',
include_config=include_config)
for line in nodelines:
if line.startswith('from'):
if line not in importlines:
importlines.append(line)
else:
lines.append(line)
# write connections
for u, _, d in flatgraph.in_edges(nbunch=node, data=True):
for cd in d['connect']:
if isinstance(cd[0], tuple):
args = list(cd[0])
if args[1] in functions:
funcname = functions[args[1]]
else:
func = create_function_from_source(args[1])
funcname = [
name for name in func.__globals__
if name != '__builtins__'
][0]
functions[args[1]] = funcname
args[1] = funcname
args = tuple([arg for arg in args if arg])
line_args = (u.fullname.replace('.', '_'), args,
nodename, cd[1])
line = connect_template % line_args
line = line.replace("'%s'" % funcname, funcname)
lines.append(line)
else:
line_args = (u.fullname.replace('.', '_'), cd[0],
nodename, cd[1])
lines.append(connect_template2 % line_args)
functionlines = ['# Functions']
for function in functions:
functionlines.append(pickle.loads(function).rstrip())
all_lines = importlines + functionlines + lines
if not filename:
filename = '%s%s.py' % (prefix, self.name)
with open(filename, 'wt') as fp:
fp.writelines('\n'.join(all_lines))
return all_lines
def hierarchy_pos(G,
root=None,
width=1.,
vert_gap=0.2,
vert_loc=0,
xcenter=0.5):
'''
From Joel's answer at https://stackoverflow.com/a/29597209/2966723
If the graph is a tree this will return the positions to plot this in a
hierarchical layout.
G: the graph (must be a tree)
root: the root node of current branch
- if the tree is directed and this is not given, the root will be found and used
- if the tree is directed and this is given, then the positions will be just for the descendants of this node.
- if the tree is undirected and not given, then a random choice will be used.
width: horizontal space allocated for this branch - avoids overlap with other branches
vert_gap: gap between levels of hierarchy
vert_loc: vertical location of root
xcenter: horizontal location of root
'''
if not nx.is_tree(G):
raise TypeError(
'cannot use hierarchy_pos on a graph that is not a tree')
if root is None:
if isinstance(G, nx.DiGraph):
root = next(iter(nx.topological_sort(
G))) #allows back compatibility with nx version 1.11
else:
root = random.choice(list(G.nodes))
def _hierarchy_pos(G,
root,
width=1.,
vert_gap=0.2,
vert_loc=0,
xcenter=0.5,
pos=None,
parent=None):
'''
see hierarchy_pos docstring for most arguments
pos: a dict saying where all nodes go if they have been assigned
parent: parent of this branch. - only affects it if non-directed
'''
if pos is None:
pos = {root: (xcenter, vert_loc)}
else:
pos[root] = (xcenter, vert_loc)
children = list(G.neighbors(root))
if not isinstance(G, nx.DiGraph) and parent is not None:
children.remove(parent)
if len(children) != 0:
dx = width / len(children)
nextx = xcenter - width / 2 - dx / 2
for child in children:
nextx += dx
pos = _hierarchy_pos(G,
child,
width=dx,
vert_gap=vert_gap,
vert_loc=vert_loc - vert_gap,
xcenter=nextx,
pos=pos,
parent=root)
return
return _hierarchy_pos(G, root, width, vert_gap, vert_loc, xcenter)
def build_recipes(recipe_list,
check_channels,
force=False,
debug=False,
n_workers=1,
worker_offset=0):
"""
build_recipes
=============
Controller method used to perpare, check, build, and process each recipe in the recipe list. It will
build a DAG with Nodes as recipes and dependencies, and edges connecting recipe to dependencies. It
removes any cyclic nodes that depend on each other. Identify new or updated recipes that need to
be built and build them.
Parameters:
-----------
1) recipe_list: (str) A list of recieps to check (The directory path of the recipe)
2) check_channels: (list) A list of channels to check against
3) force: (bool) Whether or not to force the recipe to be built even if the same verion and build exits in a channel being checked against (Default = False)
4) debug: (bool) Whether or not to run 'conda build' in the debug phase. (Default = False)
5) n_workers: (int) The number of works to use to create subdags. (Default = 1)
6) worker_offset: (int) The number to use to offset the n_workers used for subdag creation. (Default = 0)
Return:
+++++++
1) True if all recipes are checked and there are no errors. False otherwise
"""
if not recipe_list:
print(":ggd:build recipes: Nothing to be done")
return True
## create a dag
dag, name2recipe, recipe2name = build_recipe_dag(recipe_list)
if not dag:
print(":ggd:build recipes: Nothing to be done")
return True
## Remove cyclic dependencies in the build job
### If current build jobs depend on each other, can't build them
skip_dependent = defaultdict(list)
dag = remove_dag_cycles(dag, name2recipe, skip_dependent)
## Create subdag workers
subdag = get_subdags(dag, n_workers, worker_offset)
if not subdag:
print(":ggd:build recipes: Nothing to be done")
return True
print(":ggd:build recipes: {} recipes to build and test: \n{}".format(
len(subdag), "\n".join(subdag.nodes())))
## Filter recipes
filtered_recipes = [(recipe, recipe2name[recipe])
for package in nx.topological_sort(subdag)
for recipe in name2recipe[package]]
## Get the Repodata for each channel
repodata_by_channel, actualname_to_idname = get_repodata(check_channels)
## Remove defaults channel for now
if "defaults" in check_channels:
check_channels.remove("defaults")
## Check each recipe
built_recipes = []
skipped_recipes = []
failed_recipes = []
for recipe, name in filtered_recipes:
## Check if the recipe is being skipped
if name in skip_dependent:
print((
":ggd:build recipes: SKIPPING BUILD: skipping {} because it depends on {} "
" which failed build").format(recipe, skip_dependent[name]))
skipped_recipes.append(recipe)
continue
print(
":ggd:build recipes: Determining expected packages for {}".format(
recipe))
## Check a recipe to see if it is any other channel repodata and if it is if it's version/build is greater then what is in the repo data
predicted_path = check_recipe_for_build(
recipe,
check_channels,
repodata_by_channel,
actualname_to_idname,
force=force,
)
## if no predicted path, skip building this recipe
if not predicted_path:
print(":ggd:build recipes: Nothing to be done for recipe '{}'".
format(recipe))
continue
## Build the recipe
success = conda_build_recipe(recipe, check_channels, predicted_path,
debug)
## Check for a successful recipe build
if success:
built_recipes.append(recipe)
print(":ggd:build recipes: Package recipe located at {}".format(
",".join(predicted_path)))
else:
failed_recipes.append(recipe)
for pkg in nx.algorithms.descendants(subdag, name):
skip_dependent[pkg].append(recipe)
## Check for failed recipes
if failed_recipes:
print(
(":ggd:build recipes: BUILD SUMMARY: of {} recipes, "
"{} failed and {} were skipped. ").format(len(filtered_recipes),
len(failed_recipes),
len(skipped_recipes)))
if built_recipes:
print((":ggd:build recipes: BUILD SUMMARY: Although "
"the build process failed, there were {} recipes "
"built successfully.").format(len(built_recipes)))
for frecipe in failed_recipes:
print(":ggd:build recipes: BUILD SUMMARY: FAILED recipe {}".format(
frecipe))
## Purge the builds
sp.check_call(["conda", "build", "purge"],
stderr=sys.stderr,
stdout=sys.stdout)
return False
## IF not failed recipes, prompt for a successful build
print(
":ggd:build recipes: BUILD SUMMARY: SUCCESSFULLY BUILT {} of {} recipes"
.format(len(built_recipes), len(filtered_recipes)))
return True
