def scale_free_graph(n, alpha=0.41, beta=0.54, delta_in=0.2, delta_out=0):
def _choose_node(G, distribution, delta):
cumsum = 0.0
psum = float(sum(
distribution.values())) + float(delta) * len(distribution)
r = random.random()
for i in range(0, len(distribution)):
cumsum += (distribution[i] + delta) / psum
if r < cumsum:
break
return i
G = MultiDiGraph()
G.add_edges_from([(0, 1), (1, 2), (2, 0)])
gamma = 1 - alpha - beta
while len(G) < n:
r = random.random()
if r < alpha:
v = len(G)
w = _choose_node(G, G.in_degree(), delta_in)
elif r < alpha + beta:
v = _choose_node(G, G.out_degree(), delta_out)
w = _choose_node(G, G.in_degree(), delta_in)
else:
v = _choose_node(G, G.out_degree(), delta_out)
w = len(G)
G.add_edge(v, w)
return G
def scale_free_graph(n, alpha=0.41,beta=0.54,delta_in=0.2,delta_out=0):
def _choose_node(G,distribution,delta):
cumsum = 0.0
psum = float(sum(distribution.values()))+float(delta)*len(distribution)
r = random.random()
for i in range(0, len(distribution)):
cumsum += (distribution[i]+delta)/psum
if r < cumsum:
break
return i
G = MultiDiGraph()
G.add_edges_from([(0,1),(1,2),(2,0)])
gamma = 1 - alpha - beta
while len(G)<n:
r = random.random()
if r < alpha:
v = len(G)
w = _choose_node(G, G.in_degree(),delta_in)
elif r < alpha+beta:
v = _choose_node(G, G.out_degree(),delta_out)
w = _choose_node(G, G.in_degree(),delta_in)
else:
v = _choose_node(G, G.out_degree(),delta_out)
w = len(G)
G.add_edge(v,w)
return G
def _insert_pooling(graph: nx.MultiDiGraph, first_node: Node,
second_node: Node, spatial_dims):
"""
This function inserts point wise pooling layer between two nodes
"""
log.debug("STRIDE PROP: Insert pooling between {} and {}".format(
first_node.name, second_node.name))
stride_prop = second_node.stride_prop
assert len(graph.get_edge_data(first_node.id, second_node.id)) == 1
eattrs = graph.get_edge_data(first_node.id, second_node.id)[0]
graph.remove_edge(first_node.id, second_node.id)
pooling = Pooling(
graph,
dict(name='Pooling_',
spatial_dims=spatial_dims,
window=np.array([1, 1, 1, 1]),
output_spatial_shape=None,
stride=np.array(stride_prop),
pad_spatial_shape=np.array([[0, 0], [0, 0]]),
pad=np.array([[0, 0], [0, 0], [0, 0], [0, 0]]),
pool_method='max',
is_partial_inferred=False))
pooling_data = pooling.create_node_with_data([first_node])
_clean_fw_tensor_attrs(pooling_data)
graph.add_edges_from([(pooling_data.id, second_node.id, eattrs)])
def duplicate_shared_weights(graph: nx.MultiDiGraph):
"""
This function finds all const data nodes that have more that one consumer and then duplicate them
"""
data_nodes = [
Node(graph, id) for id in graph.nodes()
if Node(graph, id).soft_get('kind') == 'data'
]
for node in data_nodes:
# Check that node has const values and more than one consumer
if len(node.out_nodes()) > 1 and node.value is not None:
# Here we delete all edges between base node and it's consumers (except first), and then duplicate this
# node to connect with other consumers
while len(node.out_nodes()) > 1:
out_node = node.out_node(1)
if len(graph.get_edge_data(node.id, out_node.id)) != 1:
raise Error(
'There is more than one edge from {} node to {} node.'.
format(node.id, out_node.id))
e_attrs = graph.get_edge_data(node.id, out_node.id)[0]
graph.remove_edge(node.id, out_node.id)
data = Op.create_input_data_node(graph,
"Copy_{}".format(node.id),
np.array(node.value),
graph.node[node.id])
graph.add_edges_from([(data.id, out_node.id, e_attrs)])
def replace_pattern(self, graph: nx.MultiDiGraph, match: dict):
relu = match['relu']
reshape1 = match['reshape1']
reshape2_data = match['reshape2_data']
conv = match['conv']
if np.max(conv.pad) == 0:
return
relu_input = relu.in_node()
# Disconnect InputData-x->ReLU->Data-x->Reshape1
edge_attrs = graph.get_edge_data(relu.out_node().id, reshape1.id)[0]
graph.remove_edge(relu_input.id, relu.id)
graph.remove_edge(relu.out_node().id, reshape1.id)
# Connect InputData-->Reshape1
graph.add_edges_from([(relu_input.id, reshape1.id, edge_attrs)])
# Insert ReLU: Reshape2Data->ReLU->Data->Convolution
edge_attrs = graph.get_edge_data(reshape2_data.id, conv.id)[0]
graph.remove_edge(reshape2_data.id, conv.id)
graph.add_edges_from([(reshape2_data.id, relu.id, {
'in': 0
}), (relu.out_node().id, conv.id, edge_attrs)])
def __init__(self, F, arg):
"""
If arg is a list:
Generate the core graph corresponding to the group generated by group_gens.
If arg is a graph:
Check for validity and do all folds.
"""
assert is_FreeGroup(F), "F must be a free group"
self.F = F
self.F_rank = F.rank()
self.letter_types = range(1, self.F_rank + 1)
self.letters = list(range(-self.F_rank, 0)) + list(range(1, self.F_rank + 1))
# -r, ..., -1, 1, ..., r
if isinstance(arg, list):
group_gens = arg
assert all([gen in F for gen in group_gens]), "The generators must be elements of F."
self.group_gens = group_gens
G = MultiDiGraph()
G.add_node((0,)) # the marked vertex (id)
for i, gen in enumerate(self.group_gens):
word = gen.Tietze()
word_len = len(word)
# new_nodes = [(i, j) for j in range(1, word_len)]
# G.add_nodes_from(new_nodes)
get_node = lambda j: (0,) if (j % word_len == 0) else (i, j)
for j in range(word_len):
G.add_edge(get_node(j), get_node(j + 1), label=word[j])
G.add_edge(get_node(j + 1), get_node(j), label=-word[j])
elif isinstance(arg, MultiDiGraph):
# We are going to copy the graph, add reverse edges when needed,
# and sort the edges.
# The reason we sort the edges is to get a "canonical" version
# of the object, so subgroup_gens would be the same in different
# objects with the same graph.
G = MultiDiGraph()
G.add_nodes_from(arg.nodes())
edges = arg.edges(data='label')
G_edges = [e for e in edges]
assert len(edges) == len(arg.edges()), "Every edge has to be labelled."
for src, dst, letter in edges:
assert letter in self.letters, \
f"The edge betwen {src} and {dst} has an invalid label"
if (dst, src, -letter) not in G.edges(data='label'):
G_edges.append((dst, src, -letter))
G.add_weighted_edges_from(sorted(G_edges), weight='label')
else:
raise ValueError("arg must be a list of words or a MultiDiGraph.")
self.G = do_all_folds(G)
# The subgraph of positive edges
G_pos = MultiDiGraph()
G_pos.add_edges_from([e for e in self.G.edges(data=True) if e[2]['label'] > 0])
self.G_pos = G_pos
self.subgroup_gens = tuple(sorted(self.get_subgroup()))
def build_graph(nodes: List[Node], edges: List[Edge],
**kwargs) -> MultiDiGraph:
force_rebuild = kwargs.get("force_rebuild", False)
save_checkpoint = kwargs.get("save_checkpoint", True)
if not force_rebuild:
if os.path.exists(GRAPH_CHECKPOINT):
return load_graph()
else:
log.info("Graph checkpoint does not exist, building graph.")
graph = MultiDiGraph()
graph.add_nodes_from(nodes)
ebunch = list(map(lambda x: (x.source, x.destination, x), edges))
graph.add_edges_from(ebunch)
assert graph.number_of_nodes() == len(nodes)
assert graph.number_of_edges() == len(edges)
if save_checkpoint:
log.info("Checkpointing graph...")
with open(GRAPH_CHECKPOINT, "wb") as file:
pickle.dump(graph, file)
return graph
def __add__(self, o):
if not isinstance(o, MossNet):
raise TypeError("unsupported operand type(s) for +: 'MossNet' and '%s'" % type(o).__name__)
g = MultiDiGraph()
g.add_edges_from(list(self.graph.edges(data=True)) + list(o.graph.edges(data=True)))
g.add_nodes_from(list(self.graph.nodes(data=True)) + list(o.graph.nodes(data=True)))
return MossNet(g)
def create_data_node(graph: nx.MultiDiGraph,
op_node: Node,
attrs: dict = None,
edge_attrs: dict = None):
assert op_node is not None and op_node.kind == 'op'
assert len(op_node.out_nodes()) == 0
if attrs is None:
attrs = {}
data_node = unique_id(graph, op_node.id)
defaul_attrs = dict(kind='data',
precision="FP32",
name=data_node,
value=None,
shape=None,
data_type=None,
infer=None)
defaul_attrs.update(attrs)
graph.add_node(data_node, **add_attrs_props(defaul_attrs))
data_node = Node(graph, data_node)
if edge_attrs is not None:
graph.add_edges_from([(op_node.id, data_node.id, {
'out': 0,
**edge_attrs
})])
else:
graph.add_edges_from([(op_node.id, data_node.id, {'out': 0})])
return data_node
def replace_sub_graph(self, graph: nx.MultiDiGraph, match: dict):
node = match['op']
if not node.has_valid('bias') or (node.has_valid('bias')
and node.bias == 1):
return
# Calculate scale value & create Const op
scale_value = np.array(1. / (pow(node.bias, node.beta)))
node.alpha /= node.bias
const_node = Const(graph,
dict(value=scale_value, shape=scale_value.shape))
# Get all outputs for LRN layer
out_nodes = [node for node in node.out_nodes().values()]
# Create Mul node with inputs
mul_node = Mul(graph, dict(name=node.id + "/Mul_"))
mnode = mul_node.create_node(inputs=[node, const_node.create_node()])
# Move edges from LRN to Mul node
for out_node in out_nodes:
edge_attrs = graph.get_edge_data(node.id, out_node.id)[0]
graph.remove_edge(node.id, out_node.id)
graph.add_edges_from([(mnode.id, out_node.id, edge_attrs)])
def do_interprocedural_analyze_without_slice_criteria(
self, to_graph: nx.MultiDiGraph):
"""
图中存在外部函数调用，并且这些外部函数不包含切片准则
将函数的PDG直接嫁接到原始图表示中
function A {
call function B -- 展开
}
"""
for node_id in to_graph.nodes:
# Note: 图中当前节点为函数调用，需要进行展开
node_info = to_graph.nodes[node_id]
if "called" in node_info:
print("内部调用:{}".format(node_info))
fid = node_info["called"][0]
function_info = self.contract_info.get_function_info_by_fid(
fid)
if function_info is None:
called_function = self.contract_info.get_function_by_fid(
fid)
function_info = FunctionInfo(self.contract_info,
called_function)
# 目标函数名
called_function_name = function_info.name
# 全套大保健，需要优化
control_flow_analyzer = ControlFlowAnalyzer(
self.contract_info, function_info)
data_flow_analyzer = DataFlowAnalyzer(self.contract_info,
function_info)
inter_analyzer = InterproceduralAnalyzer(
self.contract_info, function_info)
graph_constructor = CodeGraphConstructor(
self.contract_info, function_info)
control_flow_analyzer.do_control_dependency_analyze() # 控制流分析
data_flow_analyzer.do_data_semantic_analyze() # 数据语义分析
inter_analyzer.do_interprocedural_analyze_for_state_def(
) # 过程间全局变量数据流分析
function_info.construct_dependency_graph(
) # 语义分析完之后进行数据增强，为切片做准备
graph = graph_constructor.do_code_create_without_slice() # 构图
print("内部函数合并：merge {} to {}".format(graph.graph["name"],
to_graph.graph["name"]))
graph, removed_semantic_edges = do_prepare_before_merge(
graph, called_function_name)
to_graph = do_merge_graph1_to_graph2(graph, to_graph,
node_id) # 内部函数调用展开
to_graph.add_edges_from(removed_semantic_edges)
return to_graph
def add_reshape_before_op_node(graph: nx.MultiDiGraph, data_node_name: str,
op_node_name: str, edge_attrs: dict):
"""
Adds reshape operation which expands dimension of the specified data tensor to 4D.
:param graph: graph to operate on.
:param data_node_name: the name of the data node to be reshaped to 4D tensor.
:param op_node_name: name of the TFCustomSubgraphCall node which produces the tensor.
:param edge_attrs: edge attributes which should be preserved.
:return: None
"""
data_node = Node(graph, data_node_name)
graph.remove_edge(data_node_name, op_node_name)
assert data_node['shape'] is not None
new_shape = make_shape_4d(data_node['shape'])
# reshape shape data node
reshape_shape_data_node_name = unique_id(graph, "Reshape_shape_")
graph.add_node(reshape_shape_data_node_name,
kind='data',
precision="FP32",
name=reshape_shape_data_node_name,
value=new_shape,
shape=[1])
# reshape operation node
reshape_node_name = unique_id(graph, "Reshape_")
graph.add_node(reshape_node_name,
kind='op',
precision="FP32",
type='Reshape',
name=reshape_node_name,
op='Reshape',
data_type=data_node['data_type'])
update_ie_fields(graph.node[reshape_node_name])
# reshaped data node
reshaped_value = None
if data_node['value'] is not None:
reshaped_value = np.reshape(data_node['value'], new_shape)
reshaped_data_node_name = unique_id(graph, "reshaped_data_")
graph.add_node(reshaped_data_node_name,
kind='data',
precision="FP32",
name=reshaped_data_node_name,
shape=new_shape,
value=reshaped_value,
nchw_layout=True)
graph.add_edges_from([(data_node_name, reshape_node_name, {
'in': 0
}), (reshape_shape_data_node_name, reshape_node_name, {
'in': 1
}), (reshape_node_name, reshaped_data_node_name, {
'out': 0
}), (reshaped_data_node_name, op_node_name, edge_attrs)])
def query_inductions(sample: nx.MultiDiGraph, edge_queries: List[List[Edge]],
ownership: Ownership):
remote_queries = mpi.comm.alltoall(edge_queries)
answers = [
list(filter(lambda e: e[1] in ownership, q)) for q in remote_queries
]
remote_answers = mpi.comm.alltoall(answers)
for remote_answer in remote_answers:
sample.add_edges_from(remote_answer)
def remove_useless_split_action(graph: nx.MultiDiGraph, matches: dict):
split_node = matches['split']
input = split_node.in_node(1)
output = split_node.out_node()
graph.remove_edge(input.id, split_node.id)
for u, v, d in list(graph.out_edges(output.id, data=True)):
graph.add_edges_from([(input.id, v, d)])
graph.remove_edge(u, v)
def combine_dots(dotlist,sciid):
g1=drawing.nx_agraph.read_dot(TMP+dotlist[-1])
g = MultiDiGraph()
for d in dotlist:
g2=drawing.nx_agraph.read_dot(TMP + d)
g1=compose(g1,g2)
g.add_nodes_from(g1.nodes(data=True))
g.add_edges_from(g1.edges(data=True))
g.to_directed()
g = nx_agraph.to_agraph(g)
g.write(TMP+'-'.join(sciid)+'.dot')
def combine_dots(dotlist, mathid):
g1 = drawing.nx_agraph.read_dot(TMP + dotlist[-1])
g = MultiDiGraph()
for d in dotlist:
g2 = drawing.nx_agraph.read_dot(TMP + d)
g1 = compose(g1, g2)
g.add_nodes_from(g1.nodes(data=True))
g.add_edges_from(g1.edges(data=True))
g.to_directed()
g = nx_agraph.to_agraph(g)
g.write(TMP + '-'.join(mathid) + '.dot')
def sst(n):
S = MultiDiGraph()
morphisms = [('X' + str(m + 1), 'X' + str(m), 'd' + str(m) + str(i))
for m in range(n) for i in range(m + 2)]
relations = []
if n > 1:
relations = [(('X'+str(m+2),'X'+str(m),'d'+str(m+1)+str(j)+'d'+str(m)+str(i)),\
('X'+str(m+2),'X'+str(m),'d'+str(m+1)+str(i)+'d'+str(m)+str(j-1)))
for m in range(n-1) for j in range(1,m+3) for i in range(j)]
S.add_edges_from(morphisms)
return fic(S, relations)
def combine_dots(dotlist):
g1=drawing.nx_agraph.read_dot('dots/'+dotlist[-1])
g = MultiDiGraph()
for i in xrange(len(dotlist)):
g2=drawing.nx_agraph.read_dot('dots/'+dotlist[i])
g1=compose(g1,g2)
g.add_nodes_from(g1.nodes(data=True))
g.add_edges_from(g1.edges(data=True))
g.to_directed()
g = nx_agraph.to_agraph(g)
g.write('dots/combined.dot')
def convert_add_to_scaleshift(graph: nx.MultiDiGraph):
for n in list(graph.nodes()):
node = Node(graph, n)
if node.has('op') and (node.op == 'BiasAdd' or node.op
== 'Add') and len(node.in_nodes()) == 2:
tensor_id, value_id = get_tensor_id(node), get_value_id(node)
if tensor_id is not None and value_id is not None and node.soft_get(
'can_be_scaleshift') is not False:
node['type'] = 'ScaleShift'
node['op'] = 'ScaleShift'
node.in_node(value_id).value = np.squeeze(
node.in_node(value_id).value)
node.in_node(value_id).shape = node.in_node(
value_id).value.shape
# if the node was created with eltwise then it has attribute 'operation' which should be removed from
# the IR
if node.has('operation'):
del graph.node[n]['operation']
bias_data = node.in_node(value_id)
graph[bias_data.node][node.node][0]['in'] = 2
graph[bias_data.node][node.node][0]['bin'] = 'biases'
input_data = node.in_node(tensor_id)
graph[input_data.node][node.node][0]['in'] = 0
update_ie_fields(graph.node[node.id])
weights_id = unique_id(graph, 'weights_')
graph.add_node(
weights_id,
**add_attrs_props(
dict(kind='data',
precision="FP32",
name=weights_id,
value=None,
shape=None,
data_type=None,
infer=None)))
wnode = Node(graph, weights_id)
wnode['value'] = np.full_like(bias_data.value,
1,
dtype=np.float32)
wnode['shape'] = np.array(wnode['value'].shape)
graph.add_edges_from([
(weights_id, node.node, {
'in': 1,
'bin': 'weights'
}),
])
def generate_edges(graph: nx.MultiDiGraph, transfers_df: pd.DataFrame, period_df: pd.DataFrame) -> None:
transfers_df = transfers_df[[
'route_id', 'start_stop_id', 'end_stop_id', 'duration']]
avg_transfers_df = transfers_df.groupby(['route_id', 'start_stop_id', 'end_stop_id']).mean().reset_index()
def edge_generator():
for _, route_id, start_node, end_node, duration in avg_transfers_df.itertuples():
period = period_df.at[(route_id, 'period')]
yield start_node, int(end_node), int(route_id), \
{'route_id': int(route_id), 'duration': int(duration), 'period': int(period), 'path': []}
graph.add_edges_from(edge_generator())
def to_directed(self):
"""Return a directed representation of the graph.
A new multidigraph is returned with the same name, same nodes and
with each edge (u,v,data) replaced by two directed edges
(u,v,data) and (v,u,data).
"""
from networkx import MultiDiGraph
G=MultiDiGraph()
G.add_nodes_from(self)
G.add_edges_from( ((u,v,data) for u,nbrs in self.adjacency_iter() \
for v,datalist in nbrs.iteritems() for data in datalist) )
return G
def replace_op(self, graph: nx.MultiDiGraph, node: Node):
in_node = node.in_node()
out_nodes = [node for node in node.out_nodes().values()]
graph.remove_edge(node.in_node().id, node.id)
scalar_value_op = Const(graph, dict(value=node.scalar, shape=node.scalar.shape, symbol_dict={'name': node.id + '/const'}))
add_op = Add(graph, dict(name=node.id + '/add_', symbol_dict={'name': node.id + '/add_'}))
add_node = add_op.create_node(inputs=[in_node, scalar_value_op.create_node()])
for out_node in out_nodes:
edge_attrs = graph.get_edge_data(node.id, out_node.id)[0]
graph.remove_edge(node.id, out_node.id)
graph.add_edges_from([(add_node.id, out_node.id, edge_attrs)])
return [add_node.id]
def nebula2networkx(client: GraphClient, nebula_space: str,
graph: nx.MultiDiGraph, vertex_list, edge_types):
do_simple_execute(client, 'use ' + nebula_space)
yield_statement = ",".join(
[edge_type + "._dst" for edge_type in edge_types])
seen_vertex = set()
queue_vertex = []
all_edges = {}
for v in vertex_list:
queue_vertex.insert(0, v)
while len(queue_vertex):
vertex = queue_vertex.pop()
seen_vertex.add(vertex)
get_edge_go_statement = "GO FROM {} OVER {} YIELD ".format(
vertex, ','.join(edge_types)) + yield_statement
edges_resp = client.execute_query(get_edge_go_statement)
edges = [[] for _ in edge_types]
if edges_resp.rows is not None:
for row in edges_resp.rows:
for ids, col in enumerate(row.columns):
if (col.getType()
== ttypes.ColumnValue.ID) and col.get_id() != 0:
edges[ids].append(col.get_id())
if col.get_id() not in seen_vertex:
seen_vertex.add(col.get_id())
queue_vertex.insert(0, col.get_id())
all_edges[vertex] = edges
# build networkX graph Node
vertex_info_resp = fetch_info(client, "* " + str(vertex))
vertex_info = handle_fetch_resp(vertex_info_resp)
graph.add_node(vertex,
**vertex_info[0] if len(vertex_info) > 0 else {})
# build networkX graph Edge
for vertex_src, edges in all_edges.items():
for edge_type_ids, vertexs_dst in enumerate(edges):
if len(vertexs_dst) != 0:
edge_info_fetch_statement = edge_types[
edge_type_ids] + ' ' + ','.join([
str(vertex_src) + "->" + str(dst)
for dst in vertexs_dst
])
edges_info = handle_fetch_resp(
fetch_info(client, edge_info_fetch_statement))
graph.add_edges_from([(vertex_src, vertexs_dst[i],
edges_info[i])
for i in range(len(edges_info))])
def replace_sub_graph(self, graph: nx.MultiDiGraph, match: dict):
ph = match['placeholder']
if ph.name in self.replacement_dict:
name = ph.name
if ph.has_and_set('data_type'):
data_type = ph.data_type
else:
data_type = SUPPORTED_DATA_TYPES[
graph.graph['cmd_params'].data_type][0]
string_value = self.replacement_dict[name]
try:
if data_type != np.bool:
value = np.array(string_value, dtype=data_type)
elif data_type == np.bool and graph.graph['fw'] == 'tf':
from mo.front.tf.common import tf_data_type_cast
if isinstance(string_value, list):
casted_list = list()
for v in np.array(string_value):
casted_list.append(
tf_data_type_cast[ph.data_type](v))
value = np.array(string_value, dtype=data_type)
else:
value = tf_data_type_cast[ph.data_type](string_value)
else:
raise Error("Can not cast value {} to {} data_type".format(
string_value, data_type))
except:
raise Error("Can not cast value {} to {} data_type".format(
string_value, data_type))
try:
value = np.reshape(a=value, newshape=ph.shape)
except:
raise Error("Can not reshape value {} to shape {}".format(
value, ph.shape))
out_edges = list(graph.out_edges(ph.id, data=True))
new_node = Const(graph).create_node(
attrs={
'value': value,
'data_type': type(value),
'name': name + '/const_placeholder',
'shape': ph.shape
})
erase_node(ph)
graph.add_edges_from([(new_node.id, v, attrs)
for u, v, attrs in out_edges])
log.info(
"Placeholder node \"{}\" was replaced with Const node \"{}\" with value \"{}\""
.format(name, new_node.name, value))
def add_to_graph(self, G: nx.MultiDiGraph) -> None:
"""
Adds nodes for the given relative dating to the graph.
Args:
G: the graph to work on
"""
G.add_nodes_from(self.items)
for source in self.sources:
G.add_edges_from(pairwise(self.items),
kind=self.kind,
source=source,
comments=self.comments,
dating=self,
xml=self.xmlsource,
ignore=self.ignore)
def simplify_state(state: SDFGState,
remove_views: bool = False) -> MultiDiGraph:
"""
Returns a networkx MultiDiGraph object that contains all the access nodes
and corresponding edges of an SDFG state. The removed code nodes and map
scopes are replaced by edges that connect their ancestor and succesor access
nodes.
:param state: The input SDFG state.
:return: The MultiDiGraph object.
"""
sdfg = state.parent
# Copy the whole state
G = MultiDiGraph()
for n in state.nodes():
G.add_node(n)
for n in state.nodes():
for e in state.all_edges(n):
G.add_edge(e.src, e.dst)
# Collapse all mappings and their scopes into one node
scope_children = state.scope_children()
for n in scope_children[None]:
if isinstance(n, nodes.EntryNode):
G.add_edges_from([(n, x)
for (y, x) in G.out_edges(state.exit_node(n))])
G.remove_nodes_from(scope_children[n])
# Remove all nodes that are not AccessNodes or have incoming
# wcr edges and connect their predecessors and successors
for n in state.nodes():
if n in G.nodes():
if (not isinstance(n, nodes.AccessNode) or
(remove_views and isinstance(sdfg.arrays[n.data], data.View))):
for p in G.predecessors(n):
for c in G.successors(n):
G.add_edge(p, c)
G.remove_node(n)
else:
for e in state.all_edges(n):
if e.data.wcr is not None:
for p in G.predecessors(n):
for s in G.successors(n):
G.add_edge(p, s)
G.remove_node(n)
break
return G
def dataToGraphCSV(data,
graph_type,
save_filename=None,
graph_lib='graph_tool'):
"""
Take our pandas dataframe containing all of our edge data and create the desired graph output.
:param data:
pandas data containing three columns, tweet id, author id, influenced user id
:param graph_type:
:param save_filename:
:param graph_lib:
:return:
"""
# now that we have our data, we can output
if graph_lib == 'graph_tool':
G = gt.Graph(directed=True)
G.vertex_properties['user_id'] = G.new_vertex_property('int64_t')
G.edge_properties['tweet_id'] = G.new_edge_property('int64_t')
edge_list = np.array(
[(infl_uid, auth_uid, tweet_id)
for tweet_id, infl_uid, auth_uid in data.iterrows()],
dtype=np.int64)
G.vp.user_id = G.add_edge_list(edge_list,
hashed=True,
eprops=[G.ep.tweet_id])
if save_filename is not None:
G.save(save_filename)
elif graph_lib == 'networkx':
G = MultiDiGraph(graph_type=', '.join(graph_type))
G.add_edges_from([(infl_uid, auth_uid, {
'tweet_id': tweet_id
}) for tweet_id, infl_uid, auth_uid in data.iterrows()])
if save_filename is not None:
write_graphml(G, save_filename)
elif graph_lib == 'edge_list':
G = np.array([(infl_uid, auth_uid, tweet_id)
for idx, (tweet_id, created, auth_uid,
infl_uid) in data.iterrows()],
dtype=np.int64)
return G
def to_networkx(network: Network) -> Any:
from networkx import Graph, DiGraph, MultiDiGraph, MultiGraph
if network.directed and network.multiedges:
G = MultiDiGraph()
elif not network.directed and network.multiedges:
G = MultiGraph()
elif network.directed and not network.multiedges:
G = DiGraph()
else:
G = Graph()
G.add_nodes_from([(v, network.nodes[v].attributes)
for v in network.nodes.uids])
G.add_edges_from([(network.edges[e].v.uid, network.edges[e].w.uid,
network.edges[e].attributes)
for e in network.edges.uids])
return G
def get_graph(model_spec, debug=False):
'''
Builds a networkx representation of the graph,
'''
graph = MultiDiGraph()
graph.add_nodes_from(
get_nodes(model_spec['edges']) + model_spec['legs']['in'].keys() +
model_spec['legs']['out'].keys())
graph.add_edges_from([(edge['source'], edge['target'], {
'operator': edge['operator']
}) for edge in model_spec['edges']])
if debug:
draw_graph(graph)
return graph
def extend_graph(self, graph: nx.MultiDiGraph, stops_df: pd.DataFrame) -> nx.MultiDiGraph:
first_stops = stops_df[['is_first']].reset_index('stop_id')
first_stops = list(first_stops.drop_duplicates('stop_id')['stop_id'])
start_routes = {}
for first, second, route in graph.edges:
if first not in start_routes:
start_routes[first] = set()
start_routes[first].add(route)
for start_node in first_stops:
if start_node not in start_routes:
# TODO temporary fix, nodes with names PH and PT aren't in the dict
continue
for route_id in start_routes[start_node]:
edges = self.get_edges_data_from(graph, start_node, route_id)
for edge in edges:
_, _, first_neighbour, first_duration, period = edge
graph.add_edges_from(self.extended_edges_generator(
graph, start_node, first_neighbour, first_duration, route_id, period))
return graph
def replace_sub_graph(self, graph: nx.MultiDiGraph, match: dict):
node = match['softmax']
if 'temperature' in node and node['temperature'] != 1.0:
in_node = node.in_node()
out_nodes = [node for node in node.out_nodes().values()]
graph.remove_edge(node.in_node().id, node.id)
temperature = np.array([1.0 / node.temperature])
scalar_value_op = Const(
graph,
dict(value=temperature,
shape=temperature.shape,
symbol_dict={'name': node.id + '/const'}))
mul_op = Mul(
graph,
dict(name=node.id + '/mul_',
symbol_dict={'name': node.id + '/mul_'}))
mul_node = mul_op.create_node(
inputs=[in_node, scalar_value_op.create_node()])
edge_attrs = graph.get_edge_data(node.id, out_nodes[0].id)[0]
graph.add_edges_from([(mul_node.id, node.id, edge_attrs)])
def Glob(n):
Glob = MultiDiGraph()
s_morphisms = [('X' + str(m + 1), 'X' + str(m), 's' + str(m))
for m in range(n)]
t_morphisms = [('X' + str(m + 1), 'X' + str(m), 't' + str(m))
for m in range(n)]
morphisms = s_morphisms + t_morphisms
relations = []
if n > 1:
ssts_relations = [
(('X' + str(m + 2), 'X' + str(m), 's' + str(m + 1) + 's' + str(m)),
('X' + str(m + 2), 'X' + str(m), 't' + str(m + 1) + 's' + str(m)))
for m in range(n - 1)
]
sttt_relations = [
(('X' + str(m + 2), 'X' + str(m), 's' + str(m + 1) + 't' + str(m)),
('X' + str(m + 2), 'X' + str(m), 't' + str(m + 1) + 't' + str(m)))
for m in range(n - 1)
]
relations = ssts_relations + sttt_relations
Glob.add_edges_from(morphisms)
return fic(Glob, relations)
def merge_data_nodes(graph: nx.MultiDiGraph, survived: Node, removed: Node):
if survived.has_and_set('is_output'):
graph.node[removed.id].update({'is_output': True})
for u, v, d in list(graph.in_edges(removed.id, data=True)):
graph.add_edges_from([(u, survived.id, d)])
graph.remove_edge(u, v)
for u, v, d in list(graph.out_edges(removed.id, data=True)):
graph.add_edges_from([(survived.id, v, d)])
graph.remove_edge(u, v)
for attr in graph.node[removed.id]:
if not attr in ['name']:
# We need to save debug info from removed data node
if attr == 'fw_tensor_debug_info':
if not survived.has_valid(attr):
survived[attr] = []
for fw_tensor_debug_info in removed[attr]:
survived[attr].append(fw_tensor_debug_info)
else:
survived[attr] = removed[attr]
def scale_free_graph(n, G=None,
alpha=0.41,
beta=0.54,
gamma=0.05,
delta_in=0.2,
delta_out=0,
seed=None):
"""Return a scale free directed graph
Parameters
----------
n : integer
Number of nodes in graph
G : NetworkX graph (optional)
Use as starting graph in algorithm
alpha : float
Probability for adding a new node conecgted to an existing node
chosen randomly according to the in-degree distribution.
beta : float
Probability for adding an edge between two existing nodes.
One existing node is chosen randomly according the in-degree
distribution and the other chosen randomly according to the out-degree
distribution.
gamma : float
Probability for adding a new node conecgted to an existing node
chosen randomly according to the out-degree distribution.
delta_in : float
Bias for choosing ndoes from in-degree distribution.
delta_out : float
Bias for choosing ndoes from out-degree distribution.
delta_out : float
Bias for choosing ndoes from out-degree distribution.
seed : integer (optional)
Seed for random number generator
Examples
--------
>>> G=nx.scale_free_graph(100)
Notes
-----
The sum of alpha, beta, and gamma must be 1.
Algorithm from
@article{bollobas2003dsf,
title={{Directed scale-free graphs}},
author={Bollob{\'a}s, B. and Borgs, C. and Chayes, J. and Riordan, O.},
journal={Proceedings of the fourteenth annual ACM-SIAM symposium on Discrete algorithms},
pages={132--139},
year={2003},
publisher={Society for Industrial and Applied Mathematics Philadelphia, PA, USA}
}
"""
def _choose_node(G,distribution,delta):
cumsum=0.0
# normalization
psum=float(sum(distribution.values()))+float(delta)*len(distribution)
r=random.random()
for i in range(0,len(distribution)):
cumsum+=(distribution[i]+delta)/psum
if r < cumsum:
break
return i
if G is None:
# start with 3-cycle
G=MultiDiGraph()
G.add_edges_from([(0,1),(1,2),(2,0)])
if alpha <= 0:
raise ValueError('alpha must be >= 0.')
if beta <= 0:
raise ValueError('beta must be >= 0.')
if gamma <= 0:
raise ValueError('beta must be >= 0.')
if alpha+beta+gamma !=1.0:
raise ValueError('alpha+beta+gamma must equal 1.')
G.name="directed_scale_free_graph(%s,alpha=%s,beta=%s,gamma=%s,delta_in=%s,delta_out=%s)"%(n,alpha,beta,gamma,delta_in,delta_out)
# seed random number generated (uses None as default)
random.seed(seed)
while len(G)<n:
r = random.random()
# random choice in alpha,beta,gamma ranges
if r<alpha:
# alpha
# add new node v
v = len(G)
# choose w according to in-degree and delta_in
w = _choose_node(G, G.in_degree(with_labels=True),delta_in)
elif r < alpha+beta:
# beta
# choose v according to out-degree and delta_out
v = _choose_node(G, G.out_degree(with_labels=True),delta_out)
# choose w according to in-degree and delta_in
w = _choose_node(G, G.in_degree(with_labels=True),delta_in)
else:
# gamma
# choose v according to out-degree and delta_out
v = _choose_node(G, G.out_degree(with_labels=True),delta_out)
# add new node w
w = len(G)
G.add_edge(v,w)
return G
def parseArnetminerDataset():
"""
Parse the four area dataset, and use only barebones structures to keep everything efficient.
Skips papers that:
(1)
The final parsed network
"""
inputFile = open(os.path.join(projectRoot, 'data','DBLP-citation-Feb21.txt'))
graph = MultiDiGraph()
# Sets for authors, papers, conferences, and terms found so far
indexToPaperIdMap = {}
citationCountMap = {}
indexSet = set()
beginning = inputFile.tell()
print "Parsing nodes for graph..."
# Counts for statistics
VALID_PAPERS = 1566322 # 99.62% of total papers in DBLP dataset
papersProcessed = 0
skippedPaperIndices = set()
invalidPaperIndices = set()
# Add each paper to graph (adding missing associated terms, authors, and conferences)
for title, authors, conference, terms, citationCount, index in __papersFromFile(inputFile, skippedPaperIndices, invalidPaperIndices):
# Check that index is unique, and record it
assert index not in indexSet
indexSet.add(index)
# Create unique identifier with paper index & title
paperId = '%d----%s' % (index, title)
citationCountMap[paperId] = citationCount
indexToPaperIdMap[index] = paperId
# Add symmetric edges & nodes (if they don't already exist in the network)
for author in authors:
graph.add_edges_from([(author, paperId), (paperId, author)])
graph.add_edges_from([(conference, paperId), (paperId, conference)])
for term in terms:
graph.add_edges_from([(term, paperId), (paperId, term)])
# Output progress
papersProcessed += 1
sys.stdout.write("\r Processed %d / %d papers..." % (papersProcessed, VALID_PAPERS))
# Rewind file
inputFile.seek(beginning)
print "Parsing citations for graph..."
# Counts for statistics
papersProcessed = 0
successfulCitations = 0
omittedPaperCitations = 0
invalidPaperCitations = 0
invalidCitations = 0
# Add citations to the graph
for title, index, citations in __citationsFromFile(inputFile):
citingId = '%d----%s' % (index, title)
for citationIndex in citations:
# Add citation edge if it was found
if citationIndex in indexToPaperIdMap:
successfulCitations += 1
graph.add_edge(citingId, indexToPaperIdMap[citationIndex])
# Tally missing citation appropriately
elif citationIndex in skippedPaperIndices:
omittedPaperCitations += 1
elif citationIndex in invalidPaperIndices:
invalidPaperCitations += 1
else:
print "\nCitation '%d' not found for '%s'" % (citationIndex, title)
invalidCitations += 1
# Output progress
papersProcessed += 1
sys.stdout.write("\r Processed Citations for %d / %d papers..." % (papersProcessed, VALID_PAPERS))
# Basic statistics about cleanliness of citations
totalCitations = invalidCitations + successfulCitations
successfulCitationsPercent = 100 * float(successfulCitations) / totalCitations
omittedPaperCitationsPercent = 100 * float(omittedPaperCitations) / totalCitations
invalidPaperCitationsPercent = 100 * float(invalidPaperCitations) / totalCitations
invalidCitationsPercent = 100 * float(invalidCitations) / totalCitations
print "\n\nTotal Citations: %d" % totalCitations
print " Citations Added (Successful): %d (%2.2f%%)" % (successfulCitations, successfulCitationsPercent)
print " Citations Skipped (Skipped Paper): %d (%2.2f%%)" % (omittedPaperCitations, omittedPaperCitationsPercent)
print " Citations Skipped (Invalid Paper): %d (%2.2f%%)" % (invalidPaperCitations, invalidPaperCitationsPercent)
print " Citations Invalid (Unknown): %d (%2.2f%%)" % (invalidCitations, invalidCitationsPercent)
return graph