def test_clean_null_graphs(self):
data = [ [ 1, 2, 'a' ], [ 2, 1, 'b' ], [ 2, 1, 'a' ],
[ 1, 3, 'c' ], [ 4, 5, 'd' ] ]
graph = graph_utils.build_graph_from_rows(
edgeRows=data, directed=True)
subgraph = graph_utils.get_subgraph_by_node(graph, 5)
data = networkx.node_link_data(subgraph)
self.assertEqual( data['links'], [] )
self.assertEqual( len(data['nodes']), 1 )
cleaned = graph_utils.clean_null_graph(data)
self.assertEqual( cleaned, {} )
subgraph = graph_utils.get_subgraph_by_node(graph, 4)
data = networkx.node_link_data(subgraph)
cleaned = graph_utils.clean_null_graph(data)
self.assertEqual( data, cleaned )
def data_func(group, graph):
merged_graph = networkx.DiGraph()
for node in groupNodesIndex[group]:
subgraph = graph_utils.get_subgraph_by_node(graph, node, depth=1)
merged_graph = networkx.compose(merged_graph, subgraph)
for node in groupNodesIndex[group]:
merged_graph.nodes[node]['level'] = 0
data = networkx.node_link_data(merged_graph)
cleaned = graph_utils.clean_null_graph(data)
return cleaned
def writeGraph(format, outfile): if format == 'gexf': nx.write_gexf(G, outfile) elif format == 'gml': nx.write_gml(G, outfile) elif format == 'graphml': nx.write_graphml(G, outfile) elif format == 'json': data = nx.node_link_data(G) with open(outfile, 'w') as of: json.dump(data, of) elif format == 'shp': nx.write_shp(G, outfile)
def key_graph_by_node(node, graph):
subgraph = graph_utils.get_subgraph_by_node(graph, node, track_depth=True)
data = networkx.node_link_data(subgraph)
cleaned = graph_utils.clean_null_graph(data)
return cleaned
def getSaveData(self):
return {
"edgeList": nx.node_link_data(self._model.graph),
"nodePositions": self._model.nodePositions,
"sourceNode": self._model.graph.sourceNode
}
def graph2json(graph):
return (
json.dumps(networkx.node_link_data(graph), default=json_default, skipkeys=True)
+ "\n"
)
def visualize(
network,
port=9853,
verbose=False,
config=None,
plot_in_cell_below=True,
is_test=False,
):
"""
Visualize a network interactively using Ulf Aslak's d3 web app.
Saves the network as json, saves the passed config and runs
a local HTTP server which then runs the web app.
Parameters
----------
network : networkx.Graph or networkx.DiGraph or node-link dictionary
The network to visualize
port : int, default : 9853
The port at which to run the server locally.
verbose : bool, default : False
Be chatty.
config : dict, default : None,
In the default configuration, each key-value-pair will
be overwritten with the key-value-pair provided in `config`.
The default configuration is
.. code:: python
default_config = {
# Input/output
'zoom': 1,
# Physics
'node_charge': -45,
'node_gravity': 0.1,
'link_distance': 15,
'link_distance_variation': 0,
'node_collision': True,
'wiggle_nodes': False,
'freeze_nodes': False,
# Nodes
'node_fill_color': '#79aaa0',
'node_stroke_color': '#555555',
'node_label_color': '#000000',
'display_node_labels': False,
'scale_node_size_by_strength': False,
'node_size': 5,
'node_stroke_width': 1,
'node_size_variation': 0.5,
# Links
'link_color': '#7c7c7c',
'link_width': 2,
'link_alpha': 0.5,
'link_width_variation': 0.5,
# Thresholding
'display_singleton_nodes': True,
'min_link_weight_percentile': 0,
'max_link_weight_percentile': 1
}
When started from a Jupyter notebook, this will show a
reproduced matplotlib figure of the stylized network
in a cell below. Only works if ``verbose = False``.
is_test : bool, default : False
If ``True``, the interactive environment will post
its visualization to Python automatically after 5 seconds.
Returns
-------
network_properties : dict
contains all necessary information to redraw the figure which
was created in the interactive visualization
config : dict
contains all configurational values of the interactive
visualization
"""
this_config = deepcopy(default_config)
if config is not None:
this_config.update(config)
path = netwulf_user_folder
mkdirp_customdir()
web_dir = pathlib.Path(path)
# copy the html and js files for the visualizations
prepare_visualization_directory()
# create a json-file based on the current time
file_id = "tmp_{:x}".format(int(time.time() * 1000)) + ".json"
filename = file_id
configname = "config_" + filename
filepath = str(web_dir / filename)
configpath = str(web_dir / configname)
with open(filepath, 'w') as f:
if type(network) in [nx.Graph, nx.DiGraph]:
network = nx.node_link_data(network)
if 'graph' in network:
network.update(network['graph'])
del network['graph']
json.dump(network, f, iterable_as_array=True)
with open(configpath, 'w') as f:
json.dump(this_config, f)
# change directory to this directory
if verbose:
print("changing directory to", str(web_dir))
print("starting server here ...", str(web_dir))
cwd = os.getcwd()
os.chdir(str(web_dir))
server = NetwulfHTTPServer(
("127.0.0.1", port),
NetwulfHTTPRequestHandler,
[filepath, configpath],
verbose=verbose,
)
# ========= start server ============
thread = threading.Thread(None, server.run)
thread.start()
url = "http://localhost:" + str(
port) + "/?data=" + filename + "&config=" + configname
if is_test:
url += "&pytest"
webbrowser.open(url)
try:
while not server.end_requested:
time.sleep(0.1)
is_keyboard_interrupted = False
except KeyboardInterrupt:
is_keyboard_interrupted = True
server.end_requested = True
if verbose:
print('stopping server ...')
server.stop_this()
thread.join(0.2)
posted_network_properties = server.posted_network_properties
posted_config = server.posted_config
if verbose:
print('changing directory back to', cwd)
os.chdir(cwd)
# see whether or not the whole thing was started from a jupyter notebook and if yes,
# actually re-draw the figure and display it
env = os.environ
try:
is_jupyter = 'jupyter' in pathlib.PurePath(env['_']).name
except: # this should actually be a key error
# apparently this is how it has to be on Windows
is_jupyter = 'JPY_PARENT_PID' in env
if is_jupyter and plot_in_cell_below and not is_keyboard_interrupted:
if verbose:
print('recreating layout in matplotlib ...')
fig, ax = wulf.draw_netwulf(posted_network_properties)
return posted_network_properties, posted_config
def export(self, path):
data = nx.node_link_data(self.graph)
with open(path, 'w') as fw:
]
return stop_names
def generate_distances(city_plan):
for edge in city_plan.edges():
city_plan[edge[0]][edge[1]]['distance'] = random.randint(
0, MAX_DISTANCE)
if __name__ == "__main__":
city_plan = nx.gnp_random_graph(STOPS_NUM, 0.2)
# uncomment if you want to draw it (tested only in jupyter notebook)
# plt.subplot(121)
# nx.draw(G, with_labels=True, font_weight='bold')
# add names to stops
stop_names = {
i: f"Przystanek {stop_name}"
for i, stop_name in enumerate(generate_stop_names(STOPS_NUM))
}
nx.set_node_attributes(city_plan, stop_names, 'stop_name')
# add distances between stops
generate_distances(city_plan)
# write city to file
with open("solvro_city.json", mode="w") as file:
file.write(
json.dumps(nx.node_link_data(city_plan), indent=4, sort_keys=True))
nodelist=path,
node_color='r',
node_size=40)
nx.draw_networkx_labels(g,
nodes_positions(g, [0.006, 0.006]),
labels=nodes_labels([path[0], path[-1]]),
font_size=12,
font_color='k')
nx.draw_networkx_edges(g,
nodes_positions(g),
edgelist=[(a, b) for a, b in zip(path, path[1:])],
edge_color='r',
width=1)
print_path(g, path)
plt.show()
if __name__ == '__main__':
if len(sys.argv) <= 1:
g = build_routes_graph()
graph_data = nx.node_link_data(g)
with open('routes_graph.json', 'w+') as file:
json.dump(graph_data, file)
else:
g = load_graph(sys.argv[1])
filter_isolated_nodes(g)
make_path('Kuźnice')
make_path('Morskie Oko')
make_path('Kasprowy Wierch')
def build_network(n_clicks, network_type, strain, order, detection_method,
example_data_clicks, rnaseq_contents, tnseq_contents,
rnaseq_filename, tnseq_filename):
"""Generates a network every time the make network button is clicked. Serializes results to JSON
and stores them in hidden divs. Shows download and explore network button."""
if n_clicks is None:
raise PreventUpdate
example_data = True if example_data_clicks else False
upload_msg, genes_df = parse_gene_list(rnaseq_contents, rnaseq_filename,
example_data)
genes_df.rename(columns={genes_df.columns[0]: 'gene'}, inplace=True)
gene_list = genes_df.gene.tolist()
if network_type == 'basic':
bio_network = BioNetwork(gene_list=gene_list,
strain=strain,
order=order,
detection_method=detection_method)
elif network_type == 'DE':
bio_network = DENetwork(gene_list=gene_list,
strain=strain,
order=order,
detection_method=detection_method,
de_genes_df=genes_df)
elif network_type == 'combined':
upload_msg, tnseq_genes = parse_tnseq_list(tnseq_contents,
tnseq_filename,
example_data)
bio_network = CombinedNetwork(gene_list=gene_list,
strain=strain,
order=order,
detection_method=detection_method,
de_genes_df=genes_df,
tnseq_gene_list=tnseq_genes)
else:
bio_network = None
if len(bio_network.network) == 0:
mapping_msg = dbc.Alert(
'The network is empty. Ensure that you uploaded a list of P. aeruginosa locus tags and '
'that you selected the right strain.',
color='warning',
style={'display': 'inline-block'})
enrichment_btns_display = {'display': 'none'}
else:
mapping_msg = html.Div(
'{} genes were mapped to the network out of {} genes in your list.'
.format(len(bio_network.mapped_genes),
len(bio_network.genes_of_interest)))
enrichment_btns_display = {'display': 'block'}
enrichment_options = [
{
'label':
'Full gene list ({} genes)'.format(
len(bio_network.genes_of_interest)),
'value':
'all'
},
{
'label':
'Genes mapped to network ({} genes)'.format(
len(bio_network.mapped_genes)),
'value':
'network'
},
]
json_network = json.dumps(nx.node_link_data(bio_network.network))
network_df = bio_network.network_df
genes_of_interest = bio_network.genes_of_interest
network_params = {
'strain': bio_network.strain,
'type': bio_network.network_type
}
return enrichment_btns_display, enrichment_options, 'all', mapping_msg, json_network, \
network_df.to_json(), json.dumps(network_params), json.dumps(genes_of_interest)
def get_data(self):
"""输出结果"""
temp_graph = nx.MultiDiGraph(self)
data = nx.node_link_data(temp_graph)
return data
def make_subnetwork(queried_nodes, network_df, json_str_network, strain,
network_type, low_confidence, extra_genes):
"""Returns a subnetwork using the PCSF algorithm, using the user-selected nodes as terminals."""
def make_prize_file(network_df, queried_nodes, network_type):
"""Generates .tsv file with node prizes for use with OmicsIntegrator."""
if network_type == 'gene_list':
# If there is no expression data, all prizes = 1
network_df['prize'] = 1
terminal_prizes = network_df.loc[
network_df.index.isin(queried_nodes), 'prize']
elif network_type == 'rna_seq' or network_type == 'combined':
# Set prizes to expression values
terminal_prizes = network_df.loc[
network_df.index.isin(queried_nodes), ['log2FoldChange']]
# The bigger the fold change, the bigger the prize
terminal_prizes.log2FoldChange = abs(
terminal_prizes.log2FoldChange)
terminal_prizes = terminal_prizes.rename(
columns={'log2FoldChange': 'prize'})
if network_type == 'combined':
# Set TnSeq prizes to the max prize
terminal_prizes.loc[
network_df['significanceSource'] ==
'TnSeq', :] = terminal_prizes['prize'].max()
terminal_prizes.to_csv('node_prizes.tsv', sep='\t')
network = nx.node_link_graph(json.loads(json_str_network))
# Make Graph object for prize-collecting Steiner forest (PCSF)
graph = Graph(
os.path.join(
'data',
'{}_interactome.tsv'.format(strain)), # Get interactome with costs
{
'b': 10, # b > 1 results in more terminal nodes in sub_network
'g': 0
} # g = 0 = disable degree cost correction
)
make_prize_file(network_df, queried_nodes, network_type)
graph.prepare_prizes('node_prizes.tsv')
os.remove('node_prizes.tsv'
) # Delete prize file (not needed anymore after running PCSF)
vertex_indices, edge_indices = graph.pcsf()
forest, augmented_forest = graph.output_forest_as_networkx(
vertex_indices, edge_indices)
# Include low confidence edges if selected by the user
sub_network = augmented_forest if low_confidence else forest
# If sub-network is empty, warning is shown
if len(sub_network.nodes) == 0:
return None, None
# Sub-network includes extra genes (not in the input genes)
if extra_genes:
nodes = [node for node in sub_network.nodes]
# Get extra gene information from database
with sqlite3.connect('PaIntDB.db') as db_connection:
descriptions = pd.read_sql_query("""SELECT id, product_name
FROM protein
WHERE id IN (%s)""" %
', '.join('?' * len(nodes)),
con=db_connection,
params=nodes)
short_names = pd.read_sql_query("""SELECT id, name
FROM interactor
WHERE id IN (%s)""" %
', '.join('?' * len(nodes)),
con=db_connection,
params=nodes)
# Format results to use as node attributes
descriptions = descriptions.set_index('id').to_dict(orient='index')
short_names = short_names.set_index('id').to_dict(orient='index')
description_attr = dict()
short_name_attr = dict()
for key, value in descriptions.items():
description_attr[key] = dict(description=value['product_name'])
for key, value in short_names.items():
short_name_attr[key] = dict(shortName=value['name'])
nx.set_node_attributes(sub_network, description_attr)
nx.set_node_attributes(sub_network, short_name_attr)
# Set locus tags as short names for new genes
for node in sub_network.nodes:
if sub_network.nodes[node]['shortName'] is None:
sub_network.nodes[node]['shortName'] = node
sub_network.remove_edges_from(nx.selfloop_edges(sub_network))
# Sub-network only includes genes in input genes
else:
sub_network = network.edge_subgraph(sub_network.edges())
unfrozen_sub = nx.Graph(sub_network) # Copy needed to remove orphan nodes
unfrozen_sub.remove_nodes_from(list(nx.isolates(unfrozen_sub)))
cyto_sub_network = make_cyto_elements(unfrozen_sub)
json_sub_network = json.dumps(
nx.node_link_data(unfrozen_sub)) # For downloading
return cyto_sub_network, json_sub_network
def export(self, file_path):
"""将图导出至文件"""
temp_graph = nx.MultiDiGraph(self)
data = nx.node_link_data(temp_graph)
with open(file_path, 'w', encoding='utf-8') as fw:
json.dump(data, fw)
def solve_structural_imbalance(self, subregion='Global', year=None):
"""Solves specified Stanford Militants Mapping Project structural imbalance problem and returns annotated graph.
If self._qpu is True (set during object initialization), this function will first attempt to embed the entire
problem on the hardware graph using EmbeddingComposite. Failing this, it will fallback on QBSolv to decompose
the problem. If self._qpu is False, this function will use ExactSolver for problems with less than 20 nodes.
For problems with 20 more more nodes, it will use QBSolv to solve the problem classically.
Args:
subregion (str, optional):
Filter graph by subregion. One of ['Global', 'Syria', 'Iraq']. Defaults to 'Global' (entire network).
year (int, optional):
Filter graph by year. Returns only events in or before year. Defaults to None (no filter applied).
Returns:
A dictionary with node-link formatted data. Conforms to dwave_structural_imbalance_demo.json_schema.
Optional property 'color' is set for each item in 'nodes'. Optional property 'frustrated' is set for each
item in 'links'.
"""
G_in = self._get_graph(subregion, year)
if len(G_in) == 0:
raise ValueError("Filtered network has no nodes to solve problem on")
h, J = dnx.social.structural_imbalance_ising(G_in)
# <10% of the time it will fail to find an embedding, so keep trying
while True:
try:
# use the sampler to find low energy states
response = self._sampler.sample_ising(h, J, **self._sampler_args)
break
except ValueError:
pass
# histogram answer_mode should return counts for unique solutions
if 'num_occurrences' not in response.data_vectors:
response.data_vectors['num_occurrences'] = [1] * len(response)
# should equal num_reads
total = sum(response.data_vectors['num_occurrences'])
results_dict = OrderedDict()
for sample, num_occurrences in response.data(['sample', 'num_occurrences']):
# spins determine the color
colors = {v: (spin + 1) // 2 for v, spin in iteritems(sample)}
key = tuple(colors.values())
if key in results_dict:
results_dict[key].graph["numOfOccurrences"] += num_occurrences
results_dict[key].graph["percentageOfOccurrences"] = 100 * \
results_dict[key].graph["numOfOccurrences"] / total
else:
G = G_in.copy()
# frustrated edges are the ones that are violated
frustrated_edges = {}
for u, v, data in G.edges(data=True):
sign = data['sign']
if sign > 0 and colors[u] != colors[v]:
frustrated_edges[(u, v)] = data
elif sign < 0 and colors[u] == colors[v]:
frustrated_edges[(u, v)] = data
# else: not frustrated or sign == 0, no relation to violate
for edge in G.edges:
G.edges[edge]['frustrated'] = edge in frustrated_edges
for node in G.nodes:
G.nodes[node]['color'] = colors[node]
G.graph['numOfOccurrences'] = num_occurrences
G.graph['percentageOfOccurrences'] = 100 * num_occurrences / total
results_dict[key] = G
output = {'results': [nx.node_link_data(result) for result in results_dict.values()], 'numberOfReads': total}
if 'timing' in response.info:
output['timing'] = {"actual": {"qpuProcessTime": response.info['timing']['qpu_access_time']}}
return output
def dump_json(G, filename):
directory = pathlib.Path(filename).parent
directory.mkdir(parents=True, exist_ok=True)
with open(filename, "w") as f:
f.write(json.dumps(nx.node_link_data(G), indent=4))
x[node] = r*np.cos(theta)
y[node] = r*np.sin(theta)
# write node positions to graph
nx.set_node_attributes(G, x, 'x')
nx.set_node_attributes(G, y, 'y')
# draw
fig, ax = pl.subplots(1,1, figsize=(5,4))
ax.axis('equal')
nx.draw(G,
pos={node:(x[node],y[node]) for node in G.nodes()},
node_color="#1b9e77",
edge_color="#333333",
node_size=200,
)
# save in json
node_link_data = nx.node_link_data(G)
with open('dodecahedron.json','w') as f:
json.dump(node_link_data,f,indent=2,sort_keys=True)
fig.savefig('dodecahedron.png',dpi=150)
pl.show()
"venues": {},
"cities": {},
"revues": {},
}
networks[grouped_by].finished = datetime.datetime.now()
for key in tqdm(
networks,
bar_format=
"Saving JSON files for each network: {n_fmt}/{total_fmt} {bar}",
colour="green",
):
file_name = f"{PREFIX}-co-occurrence-{key}.json"
data = nx.node_link_data(networks[key])
data["createdDate"] = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
diff = datetime.datetime.now() - networks[key].generated
data["timeToCreate"] = {
"minutes": diff.seconds // 60,
"seconds": diff.seconds % 60,
"totalInSeconds": diff.seconds,
}
data["days"] = re.findall(r"\d+", key)[0]
with open("network-app/data/" + file_name, "w+") as fp:
json.dump(obj=data, fp=fp)
if not "no-unnamed-performers" in key:
continue
def on_message(self, message):
""" Receiving a message from the websocket, parse, and act accordingly. """
msg = tornado.escape.json_decode(message)
if msg['type'] == 'config_file':
if self.application.verbose:
print(msg['data'])
self.config = list(yaml.load_all(msg['data']))
if len(self.config) > 1:
error = 'Please, provide only one configuration.'
if self.application.verbose:
logger.error(error)
self.write_message({'type': 'error', 'error': error})
return
self.config = self.config[0]
self.send_log(
'INFO.' + self.simulation_name,
'Using config: {name}'.format(name=self.config['name']))
if 'visualization_params' in self.config:
self.write_message({
'type': 'visualization_params',
'data': self.config['visualization_params']
})
self.name = self.config['name']
self.run_simulation()
settings = []
for key in self.config['environment_params']:
if type(self.config['environment_params']
[key]) == float or type(
self.config['environment_params'][key]) == int:
if self.config['environment_params'][key] <= 1:
setting_type = 'number'
else:
setting_type = 'great_number'
elif type(self.config['environment_params'][key]) == bool:
setting_type = 'boolean'
else:
setting_type = 'undefined'
settings.append({
'label': key,
'type': setting_type,
'value': self.config['environment_params'][key]
})
self.write_message({'type': 'settings', 'data': settings})
elif msg['type'] == 'get_trial':
if self.application.verbose:
logger.info('Trial {} requested!'.format(msg['data']))
self.send_log('INFO.' + __name__,
'Trial {} requested!'.format(msg['data']))
self.write_message({
'type': 'get_trial',
'data': self.get_trial(int(msg['data']))
})
elif msg['type'] == 'run_simulation':
if self.application.verbose:
logger.info('Running new simulation for {name}'.format(
name=self.config['name']))
self.send_log(
'INFO.' + self.simulation_name,
'Running new simulation for {name}'.format(
name=self.config['name']))
self.config['environment_params'] = msg['data']
self.run_simulation()
elif msg['type'] == 'download_gexf':
G = self.trials[int(msg['data'])].history_to_graph()
for node in G.nodes():
if 'pos' in G.node[node]:
G.node[node]['viz'] = {
"position": {
"x": G.node[node]['pos'][0],
"y": G.node[node]['pos'][1],
"z": 0.0
}
}
del (G.node[node]['pos'])
writer = nx.readwrite.gexf.GEXFWriter(version='1.2draft')
writer.add_graph(G)
self.write_message({
'type':
'download_gexf',
'filename':
self.config['name'] + '_trial_' + str(msg['data']),
'data':
tostring(writer.xml).decode(writer.encoding)
})
elif msg['type'] == 'download_json':
G = self.trials[int(msg['data'])].history_to_graph()
for node in G.nodes():
if 'pos' in G.node[node]:
G.node[node]['viz'] = {
"position": {
"x": G.node[node]['pos'][0],
"y": G.node[node]['pos'][1],
"z": 0.0
}
}
del (G.node[node]['pos'])
self.write_message({
'type':
'download_json',
'filename':
self.config['name'] + '_trial_' + str(msg['data']),
'data':
nx.node_link_data(G)
})
else:
if self.application.verbose:
logger.info('Unexpected message!')
def _export_to_node_link_json(self, filename: str):
"""Save the network in node-link format."""
graph = nx.node_link_data(self.graph)
return self._write_to_json(graph, filename)
def get_detail_by_ap_id(self, ap_id):
"""
Prepare the affiliated party sub-graph with additional node information
:param ap_id: the requested affiliate party id
:return: the sub-graph json
"""
# extract the requested sub-graph
ap_graph = self.tp_network.subgraph(self.ap_list[ap_id]).copy()
# retrieve taxpayer information
tp_list = self.get_taxpayer_detail(ap_graph)
# retrieve investor information
in_list = self.get_investor_detail(ap_graph)
# # Calculate suspicious value
# for in_node in in_list:
# # 根据每个investor，求她到关联交易的距离和路径数
# # node的嫌疑值 = 多条路径上的比例的相乘 的加和
# # check if ap_node conducted affiliated party transactions
# node_suspect_value = 0
# for ap_node in tp_list: # every taxpayer
# if ap_node in self.ap_network: # if taxpayer conducted related party transaction
# # 获取每一个path的invest ratio ,如果是多步就invest ratio相乘
# for path in list(nx.all_simple_paths(ap_graph, source=in_node, target=ap_node)):
# weight = 1
# for i in range(len(path)-1):
# e_dict = ap_graph.get_edge_data(path[i], path[i+1])
# weight = weight * e_dict['in_ratio'] if 'in_ratio' in e_dict else 0
# node_suspect_value += weight
# ap_graph.add_node(in_node, suspect_value=node_suspect_value)
# retrieve invoice information
ap_invoice = self.ap_txn_period.query('buyer_id in @tp_list').query(
'seller_id in @tp_list')
# obtain an undirected investment network
ap_graph_undirected = ap_graph.to_undirected()
ap_graph_undirected.remove_edges_from([
(u, v) for u, v, ap in ap_graph_undirected.edges(data='ap_txn')
if ap
])
for u, v, ap_txn in ap_graph.edges(data='ap_txn'):
if ap_txn:
# calculate the related strength of each ap_txn
related_strength = 0
txn = ap_invoice.query('buyer_id == @u').query(
'seller_id == @v')['txn_sum']
# add all simple path of each ap_txn into the graph to be highlighted
paths = list(
nx.all_simple_paths(ap_graph_undirected,
source=u,
target=v))
for path in paths:
path_strength = 1
for i in range(0, len(path) - 1):
path_strength *= ap_graph_undirected.get_edge_data(
path[i], path[i + 1])['in_ratio']
related_strength += path_strength
ap_graph.add_edge(u,
v,
ap_txn_amount=np.sum(txn),
ap_txn_count=len(txn),
path=paths,
related_strength=related_strength)
_profit = self.get_calendar_data_by_tp_id(
u, v, '2014-01-01', '2014-12-31')['profit'][-1]
ap_graph.add_node(u, profit=_profit > 0)
_profit = self.get_calendar_data_by_tp_id(
v, u, '2014-01-01', '2014-12-31')['profit'][-1]
ap_graph.add_node(v, profit=_profit > 0)
return nx.node_link_data(ap_graph)
def save(G, fname):
data = nx.node_link_data(G)
json.dump(data, open(fname, 'w'), indent=2)
def save_graph_object_to_s3(G, s3_resource, bucket_name, file_loc):
G_dict = nx.node_link_data(G)
G_json = json.dumps(G_dict)
server_message = s3_resource.Object(bucket_name, file_loc).put(Body=G_json)
return server_message
from structure import BokehStructureGraph
import json
from bokeh.plotting import figure
import networkx as nx
F = figure()
F.line(x=[1, 2, 3], y=[1, 2, 3])
K = BokehStructureGraph(F)
with open('graph_data.txt', 'w') as f:
f.write(json.dumps(nx.node_link_data(K.graph)))
KK = BokehStructureGraph(K.model)
with open('graph_data_2.txt', 'w') as f:
f.write(json.dumps(nx.node_link_data(KK.graph)))
def dump_graph(gx, filename='graph.json'):
nld = nx.node_link_data(gx)
with open(filename, 'w') as f:
dump(nld, f)
def save(self, name="DefaultGraph"):
graph_to_json = nx.node_link_data(self.graph)
#save to db anveshan resources
anveshan_resource.save_graph(graph_to_json, self.links, name)
def key_graph_by_node(node, graph):
subgraph = graph_utils.get_subgraph_by_node(graph, node, track_depth=True)
data = networkx.node_link_data(subgraph)
return data
def save(self, filename):
"""
Saves family tree's graph into a 'filename' with json format
"""
with open(filename, 'w') as file:
file.write(json.dumps(nx.node_link_data(self._graph)))
import json
import numpy as np
import networkx as nx
import matplotlib.pyplot as plt
g = nx.karate_club_graph()
fig, ax = plt.subplots(1, 1, figsize=(8, 6))
nx.draw_networkx(g, ax=ax)
with open('karate_club_graph.json', 'w') as outfile1:
outfile1.write(json.dumps(nx.node_link_data(g)))
def key_graph_by_node(node, graph):
subgraph = graph_utils.get_subgraph_by_node(graph, node, track_depth=True)
data = networkx.node_link_data(subgraph)
return data
def build_directed_graph(df,
path=work_david,
year=2000,
level='district',
graph_type='directed',
return_json=False):
import networkx as nx
import numpy as np
from networkx import NetworkXNotImplemented
"""Build a directed graph with a specific level hierarchy and year/s.
Input: df: original pandas DataFrame
year: selected year/s
level: district, county or city
return_json: convert networkx DiGraph object toJSON object and
return it.
Output: G: networkx DiGraph object or JSON object"""
print('Building {} graph with {} hierarchy level'.format(
graph_type, level))
# source = level_dict.get(level)['source']
# target = level_dict.get(level)['target']
df_sliced = choose_year(df, year=year, dropna=True)
node_sizes = node_sizes_source_target(df, year=year, level=level)
# node_geo = get_lat_lon_from_df_per_year(df, year=year, level=level)
# if weight_col is not None:
# df['weights'] = normalize(df[weight_col], 1, 10)
# else:
# df['weights'] = np.ones(len(df))
# df = df[df['Percent-migrants'] != 0]
G = create_G_with_df(df_sliced, level=level, graph_type=graph_type)
# G = nx.from_pandas_edgelist(
# df_sliced,
# source=source,
# target=target,
# edge_attr=[
# source,
# 'Percent-migrants',
# 'Direction',
# 'Number',
# 'Total',
# 'Distance',
# 'Angle'],
# create_using=Graph)
# enter geographical coords as node attrs:
geo = read_geo_name_cities(path=path)
for col in geo.columns:
dict_like = dict(
zip([x for x in G.nodes()], [geo.loc[x, col] for x in G.nodes()]))
nx.set_node_attributes(G, dict_like, name=col)
# slice df for just inflow:
df_in = df_sliced[df_sliced['Direction'] == 'inflow']
# calculate popularity index:
pi_dict = calculate_poplarity_index_for_InID(df_in)
total_dict_in = get_total_number_of_migrants(G, df_in, direction='In')
total_dict_out = get_total_number_of_migrants(G, df_in, direction='Out')
# set some node attrs:
nx.set_node_attributes(G, total_dict_in, 'total_in')
nx.set_node_attributes(G, total_dict_out, 'total_out')
total_net = {}
for (key1, val1), (key2, val2) in zip(total_dict_in.items(),
total_dict_out.items()):
assert key1 == key2
total_net[key1] = val1 - val2
nx.set_node_attributes(G, total_net, 'total_net')
# check that net totals is zero across network (conservation of people:-):
nets = []
for node in G.nodes():
nets.append(G.nodes()[node]['total_net'])
assert sum(nets) == 0
nx.set_node_attributes(G, pi_dict, 'popularity')
nx.set_node_attributes(G, node_sizes, 'size')
# nx.set_node_attributes(G, node_geo, 'coords_lat_lon')
G.name = 'Israeli migration network'
G.graph['level'] = level
G.graph['year'] = year
G.graph['density'] = nx.density(G)
try:
G.graph['triadic_closure'] = nx.transitivity(G)
except NetworkXNotImplemented as e:
print('nx.transitivity {}'.format(e))
# G.graph['global_reaching_centrality'] = nx.global_reaching_centrality(G, weight=weight_col)
# G.graph['average_clustering'] = nx.average_clustering(G, weight=weight_col)
# if weight_col is not None:
# print('adding {} as weights'.format(weight_col))
# # add weights:
# edgelist = [x for x in nx.to_edgelist(G)]
# weighted_edges = [
# (edgelist[x][0],
# edgelist[x][1],
# edgelist[x][2][weight_col]) for x in range(
# len(edgelist))]
# G.add_weighted_edges_from(weighted_edges)
print(nx.info(G))
for key, val in G.graph.items():
if isinstance(val, float):
print(key + ' : {:.2f}'.format(val))
else:
print(key + ' :', val)
# G, metdf = calculate_metrics(G, weight_col=weight_col)
if return_json:
return nx.node_link_data(G)
else:
return G
def write_obonet_gz(self, path: Union[str, pathlib.Path]) -> None:
"""Write the OBO to a gzipped dump in Obonet JSON."""
graph = self.to_obonet()
with gzip.open(path, 'wt') as file:
json.dump(nx.node_link_data(graph), file)
clean_author_handle = firstname_initial+'. '+lastname
for next_author_index, next_author_handle in enumerate(item_handle.get_authorsList()[author_index+1:]):
clean_next_author_handle = str(next_author_handle)
firstname_initial = clean_next_author_handle.split()[0][0]
lastname = clean_next_author_handle.split()[-1]
clean_next_author_handle = firstname_initial+'. '+lastname
if bib_graph.get_edge_data(clean_author_handle, clean_next_author_handle, default=0):
bib_graph[clean_author_handle][clean_next_author_handle]['weight'] = bib_graph[clean_author_handle][clean_next_author_handle]['weight'] + 1
else:
bib_graph.add_edge(clean_author_handle, clean_next_author_handle, weight = 1)
# for n in bib_graph.nodes():
# bib_graph.node[n]['weight'] = 0
#
# for n1,n2 in bib_graph.edges():
# bib_graph.node[n1]['weight'] = bib_graph.node[n1]['weight'] + nx.get_edge_attributes(bib_graph,'weight')[n1,n2]
# bib_graph.node[n2]['weight'] = bib_graph.node[n2]['weight'] + nx.get_edge_attributes(bib_graph,'weight')[n1,n2]
for n,deg in bib_graph.degree():
bib_graph.node[n]['degree'] = deg
# node_weight=[data.values()[0]*2.5 for n,data in bib_graph.nodes(data=True)]
# edge_weight = [data.values()[0] for a,b,data in bib_graph.edges(data=True)]
for n in bib_graph:
bib_graph.node[n]['name'] = n
data = nx.node_link_data(bib_graph)
with open('../output/bib-graph.json', 'w') as f:
json.dump(data, f, indent=4)
def graph_json():
"""Return a JSON string of the graph of enrollment."""
return nx.node_link_data(Enrollment.graph())
def get_trial(self, trial):
logger.info('Available trials: %s ' % len(self.trials))
logger.info('Ask for : %s' % trial)
trial = self.trials[trial]
G = trial.history_to_graph()
return nx.node_link_data(G)
def main():
# Get commandline and configuration file options
configuration: dict = initialise_options()
#
# The rest of the script acts on each item in the list of git repositories to mine
#
logging.info(
f"Start processing the {len(configuration['repo_list'])} repositories passed on"
)
for repository in configuration["repo_list"]:
logging.info(
f"--- Start processing repository {repository['owner']}/{repository['repo']} ---"
)
username = repository["owner"]
repo = repository["repo"]
########################################################################################################################################
########################################################################################################################################
# Get (all forks of) all forks
########################################################################################################################################
########################################################################################################################################
# Initialise an empty list of forks
forks = list()
forks.append({
'user': username,
'repo': repo,
'parent_user': username,
'parent_repo': repo
})
get_Github_forks(username=username,
reponame=repo,
forks=forks,
auth=configuration["auth_token"])
logging.info(f"{str(forks.__len__()-1)} forks found")
########################################################################################################################################
########################################################################################################################################
# get all commits from all previously fetched forks
########################################################################################################################################
########################################################################################################################################
known_commits: list = list(
) # compilation of all commits of all forks, without duplicates
known_commits_shas: list = list() # for easier access later
time_stamps: list = list() # so we can index commits per date
for fork in forks:
commits = list() # all commits of this fork
get_commits(username=fork['user'],
reponame=fork['repo'],
commits=commits,
config=configuration)
for commit in commits:
if not commit['commit'] in known_commits_shas:
known_commits.append(commit)
known_commits_shas.append(commit['commit'])
time_stamps.append(
np.datetime64(parser.parse(commit['CommitDate'])))
# create a panda.DataFrame with the known_commits data
sorted_commits = pd.DataFrame(list(
zip(known_commits_shas, known_commits)),
columns=['sha', 'commit_data'],
index=pd.DatetimeIndex(time_stamps))
logging.info(f"{str(known_commits.__len__())} commits found")
del known_commits, known_commits_shas, time_stamps
# convert commits to a JSON string for export
commits_JSON = json.dumps(
sorted_commits['commit_data'].values.tolist(),
sort_keys=True,
indent=4)
# save the commits to a file
output_JSON = build_export_file_path(
os.path.join(configuration["data_dir"], 'JSON_commits'),
username + '-' + repo + '.json')
with open(output_JSON, 'w') as f:
f.write(commits_JSON)
del f
# filter by time window
######################################################################################
# Arbitrary filter after April 2020 / for a test
#sorted_commits = sorted_commits[sorted_commits.index > '2020-07']
########################################################################################################################################
########################################################################################################################################
# buid the commit history based on the previously fetched (flat) list of commits
########################################################################################################################################
########################################################################################################################################
# recreate the 'network' view in GitHub (repo > insights > network)
# network is supposed to be a DAG (directed acyclic graph)
commit_history = nx.DiGraph()
build_commit_history(sorted_commits['commit_data'].values.tolist(),
commit_history)
# stringize the non string node attributes not supported by GrapML
for node in commit_history.nodes():
commit_history.nodes[node]['refs'] = str(
commit_history.nodes[node]['refs'])
commit_history.nodes[node]['parents'] = str(
commit_history.nodes[node]['parents'])
logging.info(
f"Commit history built with {len(commit_history.nodes())} nodes and {len(commit_history.edges())} edges"
)
# export the file commit history as GraphML
output_GraphML = build_export_file_path(
os.path.join(configuration["data_dir"], 'commit_histories'),
username + '-' + repo + '.GraphML')
nx.write_graphml(commit_history, output_GraphML)
################################################################################################################################################
################################################################################################################################################
# build history of file changes based on the previously previously fetched (flat) list of commits
################################################################################################################################################
################################################################################################################################################
# network is supposed to be a DAG (directed acyclic graph)
file_change_history = nx.DiGraph()
build_file_change_history(
sorted_commits['commit_data'].values.tolist(), file_change_history)
logging.info(
f"File change history built with {len([c for c in nx.connected_components(file_change_history.to_undirected())])} files and {len(file_change_history.edges())} file changes"
)
# export the file change history as GraphML
output_GraphML = build_export_file_path(
os.path.join(configuration["data_dir"], 'file_change_histories'),
username + '-' + repo + '.GraphML')
nx.write_graphml(file_change_history, output_GraphML)
################################################################################################################################################
################################################################################################################################################
# build committer graph based on the previously previously generated file change history
################################################################################################################################################
################################################################################################################################################
committer_graph = nx.MultiDiGraph()
build_committer_graph(file_change_history, committer_graph)
logging.info(
f"Commiter graph built with {len(committer_graph.nodes())} unique committers"
)
# export the file committer graph as GraphML
output_GraphML = build_export_file_path(
os.path.join(configuration["data_dir"], 'committer_graphs'),
username + '-' + repo + '.GraphML')
nx.write_graphml(committer_graph, output_GraphML)
JSON_string = json.dumps(nx.node_link_data(committer_graph),
sort_keys=True,
indent=4)
output_JSON = build_export_file_path(
os.path.join(configuration["data_dir"], 'committer_graphs'),
username + '-' + repo + '.json')
with open(output_JSON, 'w') as f:
f.write(JSON_string)
del f
output_VISJS = os.path.join(
os.path.join(configuration["data_dir"], 'committer_graphs'),
username + '-' + repo + '.html')
export_committer_graph(committer_graph, output_VISJS)
