def test_graphs(db):
pipeline = insert_pipeline(db, 'test_graphs')
dg = pipeline.to_graph()
assert len(dg.nodes) == 0
assert len(dg.edges) == 0
n1 = PipelineNode(pipeline_id=pipeline.id)
n2 = PipelineNode(pipeline_id=pipeline.id)
n1.save(db)
n2.save(db)
l1 = PipelineLink(pipeline_id=pipeline.id,
from_node_id=n1.id,
to_node_id=n2.id)
l1.save(db)
db.commit()
assert len(pipeline.nodes) == 2
assert len(pipeline.links) == 1
dg = pipeline.to_graph()
assert len(dg.nodes) == len(pipeline.nodes)
assert len(dg.edges) == len(pipeline.links)
assert dg.is_directed()
with pytest.raises(NetworkXNoCycle):
find_cycle(dg)
assert is_connected(dg.to_undirected())
def _graph(candidates, pairs):
"""
helper function for run(), evaluates winner of pairs given ordered pairs
slower than _faster_comp(), produces same results
"""
# This is the standard graph based way to do ranked pairs
g = DiGraph()
g.add_nodes_from(candidates)
# The strongest victories are added in order, unless there is a cycle,
# in which case they are skipped
edges = set()
for (i, j) in pairs:
if (j, i) not in edges:
g.add_edge(i, j)
# if a cycle exists, the edge is removed, otherwise continue
try:
find_cycle(g)
g.remove_edge(i, j)
except no_cycle:
pass
edges.add((i, j))
# We then find the source of the graph, which is the winner
winners = set()
for c in candidates:
try:
next(g.in_edges(c).__iter__())[0]
except StopIteration:
winners.add(c)
return winners
def validate_pipeline_structure(graph: Union[Graph, DiGraph]) -> bool:
assert is_connected(
graph.to_undirected()), 'The pipeline must be connected.'
try:
find_cycle(graph)
except NetworkXNoCycle:
pass
else:
raise ValueError('The pipeline must not contain a cycle.')
return True
def add_edge(self, node_a, node_b, detect_cycles=True):
"""
Overrides add_edge method to add
extra meta data to each node
"""
if node_a.id == node_b.id:
raise NoSelfLoopsError()
if (node_a.id, node_b.id) in self.edges:
raise EdgeAlreadyExistsError()
if detect_cycles:
current_graph = self
current_graph.add_edge(node_a, node_b, False)
try:
cycle = find_cycle(current_graph)
cycle_strs = ["{} -> {}".format(link[0], link[1]) for link in cycle]
raise CircularDependencyError(
"Circular dependency detected: {}".format(", ".join(cycle_strs))
)
except NetworkXNoCycle:
pass
self.add_node(node_a.id, **node_a.node_data)
self.add_node(node_b.id, **node_b.node_data)
super().add_edge(node_a.id, node_b.id)
def topological_sorted_no_elementals(self):
# if self._topological_sorted_no_elementals is not None:
# # use cached if available
# return self._topological_sorted_no_elementals
# self._topological_sorted_no_elementals = self.topological_sorted.copy()
# for code, out_degree in self.out_degree:
# if not out_degree:
# self._topological_sorted_no_elementals.remove(code)
# return self._topological_sorted_no_elementals
try:
result = self.topological_sorted.copy()
except nx.NetworkXUnfeasible:
from networkx.algorithms.cycles import find_cycle
try:
cycle = find_cycle(self)
except:
pass
else:
print("Found graph cycle:")
print(list(cycle))
raise
# collect zero out-degree (elemental) codes in a set
# to remove them in a batch, which is much faster than
# removing them from the list one at a time.
to_remove = set()
for code, out_degree in self.out_degree:
if not out_degree:
to_remove.add(code)
return [i for i in result if i not in to_remove]
def _topologically_sort_documents(self, documents):
"""Topologically sorts the DAG formed from the documents' substitution
dependency chain.
"""
documents_by_name = {}
result = []
g = networkx.DiGraph()
for document in documents:
document = document_wrapper.DocumentDict(document)
documents_by_name.setdefault((document.schema, document.name),
document)
for sub in document.substitutions:
g.add_edge((document.schema, document.name),
(sub['src']['schema'], sub['src']['name']))
try:
cycle = find_cycle(g)
except networkx.exception.NetworkXNoCycle:
pass
else:
LOG.error(
'Cannot determine substitution order as a dependency '
'cycle exists for the following documents: %s.', cycle)
raise errors.SubstitutionDependencyCycle(cycle=cycle)
sorted_documents = reversed(list(topological_sort(g)))
for document in sorted_documents:
if document in documents_by_name:
result.append(documents_by_name.pop(document))
for document in documents_by_name.values():
result.append(document)
return result
def compose_query_by_digraph(self, graph):
if self.jobs:
raise ValueError("jobs are not empty")
try:
if find_cycle(graph):
raise ValueError("cycle found in graph, not a dag")
except NetworkXNoCycle:
pass
for node in graph.nodes():
query_id = int(node)
path = eval(graph.nodes[node]["label"])
job = {
"query_id":
query_id,
"sql":
self.__get_query_string(path),
"job_config":
bigquery.QueryJobConfig(),
"dependent_query":
list(map(int, [edge[0] for edge in graph.in_edges(node)])),
"is_finished":
False,
"common_name":
path,
}
self.jobs[query_id] = job
def main():
filename = sys.argv[1]
g = ds.getData(filename)
try:
res = cyc.find_cycle(g)
print(list(res))
except nx.exception.NetworkXNoCycle:
print("[]")
def from_cytoscape(cls, cytoscape_path, graph_dir):
"""
Loads network from cytoscape but other term data from graph_dir
"""
attrs = dict(name='name', ident='id')
with open(cytoscape_path, "r") as f:
data = json.load(f)
term_graph = cls(graph_dir)
name = attrs["name"]
ident = attrs["ident"]
graph = nx.DiGraph()
#if not data.get('directed'):
# raise ValueError("Cytoscape graph must be directed!")
# add nodes
id_to_name = {}
for d in data["elements"]["nodes"]:
node_name = d["data"]["value"]
node_id = d["data"]["id"]
graph.add_node(node_name)
id_to_name[node_id] = node_name
# add edges
for d in data["elements"]["edges"]:
source = id_to_name[d["data"].pop("source")]
target = id_to_name[d["data"].pop("target")]
graph.add_edge(source, target)
if not is_directed_acyclic_graph(graph):
cycle = find_cycle(graph)
raise ValueError(f"Found cycle: {cycle}")
name_to_term = {}
for name in graph.nodes():
if name in term_graph.name_to_term:
term = term_graph.name_to_term[name].copy()
else:
term = {
"match_res": [],
"match_fns": [],
"children": [],
"name": name
}
term["children"] = list(graph.successors(name))
name_to_term[name] = term
term_graph.name_to_term = name_to_term
term_graph.dag = graph
return term_graph
def findMST(G):
F = returnTree(G)
#now we check if F is a tree (there are no cycles)
#TO ADD: also check if every node except the root node has exactly one edge entering!
if is_tree(F):
return F
else:
C = find_cycle(F)
G_prime = contract_graph(G,C)
T_prime = findMST(G_prime)
#if is_tree(T_prime):
T = extract_graph(T_prime,G_prime,C)
return T
def find_cycle_in_dgm(d):
path = list()
for node in d.nodes():
try:
cycle = find_cycle(d, source=node, orientation=None)
for source, target in cycle:
if source not in path:
path.append(source)
if target not in path:
path.append(target)
path.append(node)
return path
except NetworkXNoCycle:
continue
return None
def TTC(Gph, initial):
"""
Implements TTC given an initial ownership of houses.
initial: a dictionary with agent:house pairs.
"""
# map each house to the agent who initially owns it
house_agent_dict = {v:k for k, v in initial.items()}
# create manipulable copy of the matching graph
G = deepcopy(Gph)
nx.set_node_attributes(G, None, 'match')
# create directed graph on which the TTC algorithm runs
ttcGraph = nx.DiGraph()
agents = [(agent, data) for agent, data in G.nodes.items() if data['bipartite'] == 1]
ttcGraph.add_nodes_from(agents)
# rankings must be a sorted list for TTC - preprocess to achieve that
for n, d in ttcGraph.nodes(data=True):
d['ranking'] = listify_rankings(d['rankings'])
# run TTC
while list(ttcGraph.nodes()) != []:
# create ranking of available houses
for n, d in ttcGraph.nodes(data=True):
d['ranking'] = [i for i in d['ranking'] if house_agent_dict[i] in ttcGraph.nodes()]
# create pointers from agents to the agent who owns the house they like most
pointers = [(n, house_agent_dict(d['ranking'][0])) for n, d in all]
for start, end in pointers:
ttcGraph.add_edge(start, end)
try:
while True:
cycle = find_cycle(ttcGraph)
for start, end in cycle:
# create record of match to house
G.node[start]['match'] = initial[end]
ttcGraph.remove_node(start)
except nx.exception.NetworkXNoCycle:
pass
# return the final matching
matchG = nx.Graph()
matchG.add_nodes_from(G)
for n, d in G.nodes.items():
matchG.add_edge(n, d['match'])
return matchG
def build_graph(self):
"""
Calibrates the term graph:
For each term:
1. Removes all nonexistent children and parents
2. Adds the term as a child for all parents
3. Adds the term as a parent for all children
"""
self.dag = nx.DiGraph()
self.dag.add_nodes_from(self.name_to_term.keys())
for name, term in self.name_to_term.items():
for child_name in term.setdefault("children", []):
if child_name in self.name_to_term:
self.dag.add_edge(name, child_name)
else:
# remove nonexistent children
logging.info(f"Removing child {child_name} from {name}.")
children = term.setdefault("children", [])
children.remove(child_name)
if not is_directed_acyclic_graph(self.dag):
cycle = find_cycle(self.dag)
raise ValueError(f"Found cycle: {cycle}")
def check_cycles(self, i):
"""
Check cycles at current index (i)
Returns boolean - True if no cycles, False if cycles
"""
m, n = self.i_to_mn(i)
init_slash = self.state[m][n]
# if we put this character down, we couldn't have created a cycle
# RULE 1
if m == 0 or n == 0:
return True
# RULE 2
elif init_slash == "\\":
return True
# RULE 3
elif init_slash == "/" and self.state[m][n - 1] != "\\":
return True
# RULE 4
elif init_slash == "/": # we know self.state[m][n-1] == "\\"
if n == self.cols - 2 and self.state[m - 1][n] == "/":
return True
elif n != self.cols - 2 and self.state[
m - 1][n] == "/" and self.state[m - 1][n + 1] == "\\":
return True
if init_slash == "/":
try:
find_cycle(self.graph, (m + 1, n))
except:
return True
return False
else:
try:
find_cycle(self.graph, (m, n + 1))
except:
return True
return False
# each point in form (slash, m, n, prev_m, prev_n)
frontier = [(init_slash, m, n)]
# initialize path's found points
cur_found_coords = [(frontier[0][1], frontier[0][2])]
# new previous frontier at beginning
prev_frontier_coords = []
while len(frontier) != 0:
# for new batch
new_frontier = []
# for each value in the frontier
for val in frontier:
slash = val[0]
cur_m = val[1]
cur_n = val[2]
# find all neighbors that are not the previous
# 1
y = cur_m - 1
x = cur_n - 1
if 0 <= y < self.rows - 1 and 0 <= x < self.cols - 1:
new_slash = self.state[y][x]
# specific slash rule for number:
if new_slash == slash and slash == "\\":
new_frontier.append((new_slash, y, x))
# 2
y = cur_m - 1
x = cur_n
if 0 <= y < self.rows - 1 and 0 <= x < self.cols - 1:
new_slash = self.state[y][x]
# specific slash rule for number:
if new_slash != slash and new_slash != "-":
new_frontier.append((new_slash, y, x))
# 3
y = cur_m - 1
x = cur_n + 1
if 0 <= y < self.rows - 1 and 0 <= x < self.cols - 1:
new_slash = self.state[y][x]
# specific slash rule for number:
if new_slash == slash and slash == "/":
new_frontier.append((new_slash, y, x))
# 4
y = cur_m
x = cur_n + 1
if 0 <= y < self.rows - 1 and 0 <= x < self.cols - 1:
new_slash = self.state[y][x]
# specific slash rule for number:
if new_slash != slash and new_slash != "-":
new_frontier.append((new_slash, y, x))
# 5
y = cur_m + 1
x = cur_n + 1
if 0 <= y < self.rows - 1 and 0 <= x < self.cols - 1:
new_slash = self.state[y][x]
# specific slash rule for number:
if new_slash == slash and slash == "\\":
new_frontier.append((new_slash, y, x))
# 6
y = cur_m + 1
x = cur_n
if 0 <= y < self.rows - 1 and 0 <= x < self.cols - 1:
new_slash = self.state[y][x]
# specific slash rule for number:
if new_slash != slash and new_slash != "-":
new_frontier.append((new_slash, y, x))
# 7
y = cur_m + 1
x = cur_n - 1
if 0 <= y < self.rows - 1 and 0 <= x < self.cols - 1:
new_slash = self.state[y][x]
# specific slash rule for number:
if new_slash == slash and new_slash == "/":
new_frontier.append((new_slash, y, x))
# 8
y = cur_m
x = cur_n - 1
if 0 <= y < self.rows - 1 and 0 <= x < self.cols - 1:
new_slash = self.state[y][x]
# specific slash rule for number:
if new_slash != slash and slash != "-":
new_frontier.append((new_slash, y, x))
# get rid of ones from previous generation
news = []
for new in new_frontier:
if (new[1], new[2]
) not in prev_frontier_coords and new not in frontier:
news.append(new)
new_frontier = news
# check that there are no duplicates found
to_set = [(i[1], i[2]) for i in new_frontier]
if len(to_set) != len(set(to_set)):
return False
# check that none of the new_frontier have been found before in path
for new in new_frontier:
if (new[1], new[2]) in cur_found_coords:
return False
# update frontier
prev_frontier_coords = [(i[1], i[2]) for i in frontier]
cur_found_coords = [(i[1], i[2]) for i in new_frontier]
frontier = new_frontier
pass # end of while(frontier) loop
return True
def _topologically_sort_documents(self, substitution_sources):
"""Topologically sorts the DAG formed from the documents' layering
and substitution dependency chain.
"""
result = []
def _get_ancestor(doc, parent_meta):
parent = self._documents_by_index.get(parent_meta)
# Return the parent's replacement, but if that replacement is the
# document itself then return the parent.
use_replacement = (
parent and parent.has_replacement and
parent.replaced_by is not doc
)
if use_replacement:
parent = parent.replaced_by
return parent
g = networkx.DiGraph()
for document in self._documents_by_index.values():
if document.parent_selector:
# NOTE: A child-replacement depends on its parent-replacement
# the same way any child depends on its parent: so that the
# child layers with its parent only after the parent has
# received all layering and substitution data. But other
# non-replacement child documents must first wait for the
# child-relacement to layer with the parent, so that they
# can use the replaced data.
parent_meta = self._parents.get(document.meta)
ancestor = _get_ancestor(document, parent_meta)
if ancestor:
g.add_edge(document.meta, ancestor.meta)
for sub in document.substitutions:
# Retrieve the correct substitution source using
# ``substitution_sources``. Necessary for 2 reasons:
# 1) It accounts for document replacements.
# 2) It effectively maps a 2-tuple key to a 3-tuple document
# unique identifier (meta).
src = substitution_sources.get(
(sub['src']['schema'], sub['src']['name']))
if src:
g.add_edge(document.meta, src.meta)
try:
cycle = find_cycle(g, orientation='reverse')
except networkx.exception.NetworkXNoCycle:
pass
else:
LOG.error('Cannot determine substitution order as a dependency '
'cycle exists for the following documents: %s.', cycle)
raise errors.SubstitutionDependencyCycle(cycle=cycle)
sorted_documents = reversed(list(topological_sort(g)))
for document_meta in sorted_documents:
if document_meta in self._documents_by_index:
result.append(self._documents_by_index[document_meta])
for document in self._documents_by_index.values():
if document not in result:
result.append(document)
return result
def SPR(admixture_graph,
force_draw_node=None,
print_output=False,
locate_cycle=True):
""" Subtree pruning and regrafting
A node is drawn and the non-admixed parent of this node is detached, and
The broken edge is reconnected, and finally the detached node is inserted
to a branch.
"""
# random.seed(101)
# TO DO: assert t_event is in the node attribute
ag = admixture_graph.copy()
current_attributes = dict(ag.nodes(data=True))
original_edges = ag.edges
if print_output:
plot_graph(ag, 'original graph')
print(f'events: {ag.get_events()}')
# detach a subtree and reconnect the broken part
candidate_pruned_nodes = get_pruned_node_candidate(ag)
if print_output:
print(f'candidate_pruned_nodes: {candidate_pruned_nodes}')
if force_draw_node:
pruned_node = force_draw_node
else:
pruned_node = sample(candidate_pruned_nodes, 1)[0]
parent_pruned = ag.parent_merge_node(pruned_node)
if print_output:
print(f'pruned: {pruned_node}; its parent {parent_pruned}')
reconnect_node_parent = next(ag.predecessors(parent_pruned))
reconnect_node_child = ag.successors(parent_pruned)
reconnect_node_child = [
r for r in reconnect_node_child if r != pruned_node
][0]
lower_bound_node = ag.lower_bound_node(pruned_node) # for lower bound
lower_bound_time = ag.get_event_time(lower_bound_node)
if print_output:
print(f'lower bound node : {lower_bound_node}')
t0 = ag.get_event_time(parent_pruned) - ag.get_event_time(lower_bound_node)
ag.remove_node(parent_pruned)
ag.add_edge(reconnect_node_parent, reconnect_node_child)
if print_output:
print(f'reconnet edge({reconnect_node_parent},{reconnect_node_child})')
reconnected_nodes = (reconnect_node_parent, reconnect_node_child)
if print_output:
plot_graph(ag, 'detached graph')
# find a brach to insert the detached subtree
# 1) pruned subtree must be younger than the inserted branch
# 2) must draw a different branch
# 3) no cycle formed
subgraph = find_target_subgraph(ag, pruned_node)
cycle = True
while cycle:
ag_holder = ag.copy()
branch_to_attach = draw_random_branch(subgraph)
branch_to_attach = ('RT', 'D')
condition_event_time = True
condition_same_branch = True
while condition_event_time | condition_same_branch:
branch_to_attach = draw_random_branch(subgraph)
if branch_to_attach not in original_edges:
branch_to_attach = branch_to_attach[::-1]
condition_event_time = (ag_holder.get_event_time(
branch_to_attach[0]) < lower_bound_time)
# print(f'condition_event_time:{condition_event_time}')
condition_same_branch = (branch_to_attach == reconnected_nodes)
# print(f'condition_same_branch: {condition_same_branch}')
# print(f'branch to attach: {branch_to_attach}')
ag_holder.remove_edge(*branch_to_attach)
ag_holder.add_edge(branch_to_attach[0], parent_pruned)
ag_holder.add_edge(parent_pruned, branch_to_attach[1])
# debug
ag_holder.add_edge(parent_pruned, lower_bound_node)
try:
c = find_cycle(ag_holder)
if locate_cycle:
print(
f'cycle:{c}; prune node {pruned_node}; branch to attach: {branch_to_attach}'
)
#plot_graph(ag_holder, 'cycle')
cycle = True
# return ag_holder
except nx.exception.NetworkXNoCycle:
cycle = False
print(cycle)
ag = ag_holder
t1 = ag.get_event_time(branch_to_attach[0]) - ag.get_event_time(
branch_to_attach[1])
if print_output:
print(
f'add edge: {(branch_to_attach[0],parent_pruned)},{(parent_pruned, branch_to_attach[1])}, {(parent_pruned, lower_bound_node)}'
)
nx.set_node_attributes(ag,
{parent_pruned: current_attributes[parent_pruned]})
# set random event time
lower = max(ag.get_event_time(branch_to_attach[1]),
ag.get_event_time(lower_bound_node))
upper = ag.get_event_time(branch_to_attach[0])
t = np.random.uniform(lower, upper)
ag.set_event_time({parent_pruned: t})
if print_output:
print(f'lower_upper = ({lower},{upper})')
print(f'ag.set_event_time({parent_pruned}:{t})')
print(
f'detach node {pruned_node}; regraft to branch {branch_to_attach}')
plot_graph(ag, 'graph after SPR')
return ag, t0 / t1
def find_cycle_(graph, cellid):
try:
cycle = [c[0] for c in find_cycle(graph, cellid)]
except NetworkXNoCycle as nc:
cycle = []
return cycle
def mif_main(g: MultiGraph, f: set, t, k: int) -> set: k_set = k != None new_k1 = new_k2 = None if k_set and k > g.order(): return None if f == g.nodes() or (k_set and k <= 0): return f if (not f): g_degree = g.degree() g_max_degree_node = max(g_degree, key=lambda n: g_degree[n]) if (g_degree[g_max_degree_node] <= 1): return set(g.nodes()) else: fx = f.copy() fx.add(g_max_degree_node) gx = g.copy() gx.remove_node(g_max_degree_node) if k_set: new_k1 = k-1 new_k2 = k mif_set1 = mif_preprocess_1(g, fx, t, new_k1) mif_set2 = mif_preprocess_1(gx, f, t, new_k2) if not mif_set1: return mif_set2 elif not mif_set2: return mif_set1 else: return max(mif_set1, mif_set2, key=len) # Set t as active vertex if t == None or not t in f: t = next(iter(f)) gd_over_3 = None gd_2 = None for v in g.neighbors_iter(t): (gd_v, gn_v) = generalized_degree(g, f, t, v) if gd_v <= 1: f.add(v) if k_set: new_k1 = k-1 return mif_preprocess_1(g, f, t, new_k1) elif gd_v >= 3: gd_over_3 = v else: gd_2 = (v, gn_v) if gd_over_3 != None: # Cannot simply use "if gd_over_3" because v might be 0 fx = f.copy() fx.add(gd_over_3) gx = g.copy() gx.remove_node(gd_over_3) if k_set: new_k1 = k-1 new_k2 = k mif_set1 = mif_preprocess_1(g, fx, t, new_k1) mif_set2 = mif_preprocess_1(gx, f, t, new_k2) if not mif_set1: return mif_set2 elif not mif_set2: return mif_set1 else: return max(mif_set1, mif_set2, key=len) elif gd_2 != None: (v, gn) = gd_2 fx1 = f.copy() fx2 = f.copy() fx1.add(v) for n in gn: fx2.add(n) gx = g.copy() gx.remove_node(v) if k_set: new_k1 = k-2 new_k2 = k-1 try: cyc.find_cycle(gx.subgraph(fx2)) mif_set1 = None except: mif_set1 = mif_preprocess_1(gx, fx2, t, new_k1) mif_set2 = mif_preprocess_1(g, fx1, t, new_k2) if not mif_set1: return mif_set2 elif not mif_set2: return mif_set1 else: return max(mif_set1, mif_set2, key=len) return None