def renderPage(self, extension): uid = self.request.get('uid', None) network = None strpage = "Error." supported_extensions = ['gdf', 'graphml'] if not uid == None and extension in supported_extensions: session = sessionmanager.getsession(self) if session and session['me']['id'] == uid: q = db.GqlQuery("SELECT * FROM Network WHERE uid = :1", uid) networks = q.fetch(1) if not len(networks) == 0: network = networks[0] graph = loadGraph(network.getnodes(), network.getedges()) strpage = '' if extension == 'gdf': for line in nx.generate_gdf(graph): strpage += line + '\n' self.response.headers['Content-Type'] = "text/gdf" if extension == 'graphml': for line in nx.generate_graphml(graph): strpage += line + '\n' self.response.headers['Content-Type'] = "xml/graphml" self.response.out.write(strpage)
def at_vdt_etg(limit=None): at_vdts_map = construct_at_dt_map_from_file(AT_VDTS_FILENAME) with open(AT_VDT_ETG_FILENAME, mode='w') as at_vdt_eth_file: writer = csv.writer(at_vdt_eth_file, delimiter=DELIMITER, quotechar=ETG_QUOTE_CHAR, quoting=csv.QUOTE_MINIMAL) at_vdts_size = len(at_vdts_map) percentage = 0 page_count = 0 init_datetime = datetime.datetime.now() for ambiguation_term_title, valid_disambiguation_terms in at_vdts_map.items( ): row_items = [] for vdt in valid_disambiguation_terms: row_items.append(ambiguation_term_title) row_items.append(vdt.title()) etg = get_etg(vdt) etg_grapml = list(generate_graphml(etg, prettyprint=False))[0] row_items.append(etg_grapml) writer.writerow(row_items) row_items = [] page_count += 1 percentage = (page_count * 100.0) / at_vdts_size curr_time = datetime.datetime.now() date_time = curr_time - init_datetime print(date_time) print("% [", percentage, "] of pages processed.", " ", date_time, " has passed.") etc = (date_time / percentage) * (100 - percentage) print(" ETC: ", etc)
def renderPage(self, extension): uid = self.request.get("uid", None) network = None strpage = "Error." supported_extensions = ["gdf", "graphml"] if not uid == None and extension in supported_extensions: session = sessionmanager.getsession(self) if session and session["me"]["id"] == uid: q = db.GqlQuery("SELECT * FROM Network WHERE uid = :1", uid) networks = q.fetch(1) if not len(networks) == 0: network = networks[0] graph = loadGraph(network.getnodes(), network.getedges()) strpage = "" if extension == "gdf": for line in nx.generate_gdf(graph): strpage += line + "\n" self.response.headers["Content-Type"] = "text/gdf" if extension == "graphml": for line in nx.generate_graphml(graph): strpage += line + "\n" self.response.headers["Content-Type"] = "xml/graphml" self.response.out.write(strpage)
def main(out_dir, direction, num_graphs, graph_params, eta, require_connected=True, require_invertible=True, require_stationary=True): graph_params = yaml.safe_load(graph_params) graphs = dict() for i in range(num_graphs): i_d = str(uuid4()) graph = make_graph(graph_params, eta, require_connected=require_connected, require_invertible=require_invertible, require_stationary=require_stationary) graphs[i_d] = '\n'.join(nx.generate_graphml(graph)) out_file = Path(out_dir) / 'undirected_sw_graphs.yaml' out_file.write_text(yaml.dump(graphs)) out_path = Path(out_dir) / 'watts_strogats' out_path.mkdir(parents=True, exist_ok=True) graph_params_str = '_'.join([ f'{param}_{graph_params[param]}' for param in sorted(graph_params.keys()) ]) out_file = out_path / f'{direction}_{graph_params_str}.yaml' out_file.write_text(yaml.dump(graphs))
def main(out_dir, direction, num_graphs, num_nodes, graph_params, eta, require_connected=True, require_invertible=True, require_stationary=True): graph_params = yaml.safe_load(graph_params) graphs = dict() i = 0 while i < num_graphs: i_d = str(uuid4()) graph_params['n'] = num_nodes graph = make_random_graph(graph_params, eta, require_connected=require_connected, require_invertible=require_invertible, require_stationary=require_stationary) graphs[i_d] = '\n'.join(nx.generate_graphml(graph)) i += 1 out_path = Path(out_dir) / 'erdos_renyi' out_path.mkdir(parents=True, exist_ok=True) out_file = out_path / f'{direction}_p_{graph_params["p"]}.yaml' out_file.write_text(yaml.dump(graphs))
def write_service_graphs(service): graphsdir = 'graphs' try: os.makedirs(graphsdir) except OSError as exc: if exc.errno == errno.EEXIST and os.path.isdir(graphsdir): pass service.build_topology_graph(level=3, bridges=False) for lvl in range(0, 4): g = service.build_topology_graph(level=lvl, bridges=False) nx.write_graphml( g, os.path.join(graphsdir, "{0}-lvl{1}.graphml".format(service.id, lvl))) g = service.build_topology_graph(level=lvl, bridges=True) nx.write_graphml( g, os.path.join(graphsdir, "{0}-lvl{1}-br.graphml".format(service.id, lvl))) g = service.build_topology_graph(level=3, bridges=True, vdu_inner_connections=False) service.complete_graph = nx.generate_graphml(g, encoding='utf-8', prettyprint=True) nx.write_graphml( g, os.path.join(graphsdir, "{0}-lvl3-complete.graphml".format(service.id)))
def hashtag_user_graph(self, request, queryset): graph = Tweet.objects.hashtag_user_graph(queryset) response = HttpResponse( '\n'.join(nx.generate_graphml(graph)), content_type='text/graphml' ) response['Content-Disposition'] = 'attachment; filename=hashtags-users.graphml' return response
def download_common_followers_graph(self, request, queryset): graph = Community.objects.build_common_followers_graph(queryset) response = HttpResponse( '\n'.join(nx.generate_graphml(graph)), content_type='text/csv' ) response['Content-Disposition'] = 'attachment; filename="common-followers-%s.graphml"' % '-'.join([c.name for c in queryset]) return response
def to_graphml(nodes: List[str], edges: List[Edge]) -> str: G = nx.Graph() G.add_nodes_from(nodes) G.add_edges_from([(e.source, e.target) for e in edges]) graph_str = "\n".join(nx.generate_graphml(G, prettyprint=False)) return graph_str
def getdag(request): name = request.GET.get('name', '') if (name == ''): return HttpResponse("DAG not specified.") logger.info('WORKFLOW: Received request for DAG %s', name) g = fetchdag(name) s = '\n'.join(nx.generate_graphml(g)) return HttpResponse(s)
def query(self, opts): if opts.log_level: LOGGER.setLevel(opts.log_level) if opts.id: # if opts.id is provided, query a egocentric network nodes, links = self.egocentric(opts) else: # otherwise a sampled network nodes, links = self.sociocentric(opts) G = self.generateGraph([{'id': n} for n in nodes], links, opts) self.__debugGraph(G) self.densifyGraph(G, opts.limit) self.__debugGraph(G) node_data, metrics = self.getGraphDetails(G, opts) # if optimize>1, removed nodes causing trouble try: for ob in node_data: n = ob.get('id') if n: for k,v in ob.items(): if k!='id': G.nodes[n][k] = v else: LOGGER.debug("No 'id' found for {}".format(ob)) except Exception as e: LOGGER.error("{} occured".format(e)) LOGGER.error("{}".format(node_data)) raise e ''' Check if coordinates need adjusting based on ?_x or ?_y result value ''' for _,v in G.nodes(data=True): if ('_x' in v.keys() or '_y' in v.keys()): LOGGER.debug("self.adjustPositions") self.adjustPositions(G) break if opts.format == NetworkBuilder.GRAPHML: res = '\n'.join(nx.generate_graphml(G, prettyprint=True)) else: res = nx.readwrite.json_graph.cytoscape_data(G) res['metrics'] = metrics return res
async def respond_graphml( gen: AsyncIterator[JsonElement]) -> AsyncGenerator[str, None]: # Note: this is a very inefficient way of creating a response, since it creates the graph in memory # on the server side, so we can reuse the networkx code. # This functionality can be reimplemented is a streaming way. def no_nested_props(js: Json) -> Json: reported: Json = value_in_path_get(js, NodePath.reported, {}) res = { k: v for k, v in reported.items() if v is not None and not isinstance(v, (dict, list)) } return res graph = await result_to_graph(gen, no_nested_props) for line in generate_graphml(graph): yield line
def serialize_graph(self, format: GraphFormat = GraphFormat.GRAPHML) -> str: """ Serialize a given graph into GraphML string or return None if graph is not found :return: """ graph = self.storage.extract_graph(self.graph_id) graph_string = None if graph is not None: if format == GraphFormat.GRAPHML: graph_string = '\n'.join(nx.generate_graphml(graph)) elif format == GraphFormat.JSON_NODELINK: json_object = nx.readwrite.node_link_data(graph) graph_string = json.dumps(json_object) elif format == GraphFormat.CYTOSCAPE: json_object = nx.readwrite.cytoscape_data(graph) graph_string = json.dumps(json_object) else: raise PropertyGraphQueryException( graph_id=self.graph_id, node_id=None, msg=f"Unsupported export graph format {format.name}") return graph_string
def get_graphml_for_culture(self, node_list): trait_subgraph = self.graph.subgraph(node_list) linefeed=chr(10) return linefeed.join(nx.generate_graphml(trait_subgraph))
def get_graphml_for_culture(self, node_list): trait_subgraph = self.graph.subgraph(node_list) linefeed = chr(10) return linefeed.join(nx.generate_graphml(trait_subgraph))
# -*- coding: utf-8 -*- """ Created on Sat Jun 30 22:19:24 2018 @author: Ran_the_User """ import networkx as nx #import matplotlib.pyplot as plt #%matplotlib wx G = nx.readwrite.graphml.read_graphml('bokeh_server_flow.graphml') nx.draw(G, with_labels=True, node_size=700, node_shape='p') # one of 'so^>v<dph8'. #G.nodes() linefeed = chr(10) # linefeed=\n s = linefeed.join(nx.generate_graphml(G)) s = nx.parse_graphml
def to_graphml(self, output_path="data/graph.ml"): with open(output_path, "w") as f: f.write(nx.generate_graphml(compact_graph(self), encoding="utf-8"))
def centrality(): start_time = datetime.now() #TODO add config file read #TODO support cross network calculations (author_node --is--> author_node) ## >Get the REQUIRED parameters req_params = {} for entry in req_param_list: if request.args.get(entry) is not None: req_params[entry] = urllib2.unquote(request.args.get(entry)).replace('\'', '') else: ret_string = {'error': 'Required parameter missing: ' + entry} inf_sup.append_to_log(log_filename, str(ret_string)) return jsonify(ret_string) #TODO Validate start_date, end_date ## >Verify the metric is valid if req_params['metric'] not in metric_list: ret_string = {'error': 'Invalid metric requested'} inf_sup.append_to_log(log_filename, str(ret_string)) return jsonify(ret_string) ## >Verify the start date is before the end date if int(req_params['start_date']) > int(req_params['end_date']): ret_string = {'error': 'End data before start date'} inf_sup.append_to_log(log_filename, str(ret_string)) return jsonify(ret_string) ## >Get the OPTIONAL parameters opt_params = {} for entry in opt_param_list: if request.args.get(entry) is not None: opt_params[entry] = urllib2.unquote(request.args.get(entry)).replace('\'', '') else: opt_params[entry] = None #TODO validate the optional parameters ## >Get the FORMAT parameters for_params = {} for entry in format_param_list: if request.args.get(entry) is not None: for_params[entry] = urllib2.unquote(request.args.get(entry)).replace('\'', '') else: for_params[entry] = None params = dict(req_params.items() + opt_params.items() + for_params.items()) ## >Build the mongo query mongo_query = {} mongo_query['PostDate'] = {'$gte': params['start_date'], '$lte': params['end_date']} mongo_query['Network'] = params['network'] for param, value in opt_params.iteritems(): if value is not None: if param is 'type': mongo_query['Type'] = opt_params['type'] if param is 'twit_collect': mongo_query['Meta.sources'] = {'$in': [opt_params['twit_collect']]} if param is 'matched_project': mongo_query['Matching'] = {'$elemMatch': {'ProjectId': opt_params['matched_project']}} if param is 'matched_topic': #TODO pass if param is 'scored_project': #TODO pass if param is 'scored_topic': #TODO pass ## >Check if there are any matches if author_collection.find(mongo_query).count == 0: ret_string = {'error': 'No connections found matching the criteria'} inf_sup.append_to_log(log_filename, str(ret_string)) return jsonify(ret_string) else: ## >Map/reduce the A-->A connections a2a_map = Code(""" function () { emit({"author": this.Author, "connection": this.Connection}, {"count": 1} ); } """) a2a_reduce = Code(""" function (key, values) { var count = 0; values.forEach(function(v) { count += v['count']; }); return {"count": count}; } """) a2a_result = author_collection.map_reduce(a2a_map, a2a_reduce, "a2a_results", query=mongo_query).find() ## >Build the author list author_list = [] for a2a_count in a2a_result: con_author = a2a_count['_id']['author'].replace('&', '&') con_connect = a2a_count['_id']['connection'].replace('&', '&') if (len(con_author) > 0) and (len(con_connect) > 0): author_list.append((con_author, con_connect, int(a2a_count['value']['count']))) ## >Influence Calculations if len(author_list) > 0: ## >Create a black graph G = nx.DiGraph() ## >Add the endges to the graph G.add_weighted_edges_from(author_list) ## >Run the requested metric, on the graph 'G' try: calc_metric, stats = inf.run_metric(params['metric'], G, 'weight', True) except: try: if params['metric'] is 'pagerank': calc_metric, stats = inf.run_metric('pagerank_norm', G, 'weight', True) else: return jsonify({'error': 'Error calculating metric'}) except: return jsonify({'error': 'Pagerank did not converge'}) else: ret_string = {'error': 'No connections found matching the criteria'} inf_sup.append_to_log(log_filename, str(ret_string)) return jsonify(ret_string) ## >Build the dictionary to return data_results = {} ## >Append the metric data data_results['metrics'] = calc_metric ## >If graph requested if for_params['return_graph'] is not None: if for_params['return_graph'].lower() == 'true': ## >If format = data if for_params['format'] is None: ## >Append the graph data data_results['graph'] = nx.to_edgelist(G, nodelist=None) ## >If format = graphml elif for_params['format'].lower() == 'graphml': ## >Create the graphml filename graphml_name = inf_sup.create_filename(params) ## >Get the graphml data graphml_data = '\n'.join(nx.generate_graphml(G)) ## >Add the versioning graphml_final = '<?xml version="1.0" encoding="UTF-8"?>' + "\n" h = HTMLParser.HTMLParser() for line in graphml_data.split("\n"): ## >Escape the html content line = h.unescape(line) ## >For each node add appropriate metric data into the graphml if '<node id="' in line: graphml_final += (line.replace('/>', '>') + "\n") node_name = line.partition('"')[-1].rpartition('"')[0] graphml_final += ' <data key="d1">' + str(calc_metric[node_name]) + '</data>' + "\n" graphml_final += ' </node>' + "\n" else: graphml_final += line + "\n" ## >Add the key for the metric attribute if '<key' in line: graphml_final += ' <key attr.name="' + params['metric'] + '" attr.type="float" for="node" id="d1" />' if app.debug is True: ## >Write out the graphml for testing graphml_name = inf_sup.create_filename(params) with open(graphml_name, 'w') as output_file: for line in graphml_final: output_file.write(line.encode('utf-8')) if not output_file.closed: output_file.close() ## >Create the appropriate response to return the graphml response = make_response(graphml_final) response.headers["Content-Type"] = 'text/xml' response.headers["Content-Distribution"] = 'attachment; filename=%s' % (graphml_name,) return response ## >To the log statistics = {} statistics['api_query'] = params statistics['mongo_query'] = mongo_query statistics['influence_metric'] = params['metric'] statistics['metric_runtime'] = stats statistics['full_runtime'] = str(datetime.now() - start_time) statistics['graph_nodes'] = G.order() statistics['graph_edges'] = G.size() inf_sup.append_to_log(log_filename, str(statistics)) if app.debug is True: ### >Write out the influence for testing graphml_name = inf_sup.create_filename(params) influence_file = graphml_name.replace('.graphml', '.txt') with open(influence_file, 'w') as output_file: graph_list = calc_metric.items() for item in graph_list: output_file.write(item[0].encode('utf_8') + "," + str(item[1]) + '\n') if not output_file.closed: output_file.close() return jsonify(result=data_results)
def to_graphml(self): # Stringify all entities in graph, because GraphML exporter doesn't like non-string objects. g = nx.DiGraph() for s, p, e in self.relations(): g.add_edge(str(s), str(p), str(e)) return nx.generate_graphml(g)
def draw(gr, impacts={}, changes={}, return_svg=False, return_graphml=False, save_svg=True, save_graphml=False): """ Use GraphViz/Dot to draw a topology, optionally highlighting elements of interest to ensure they stand out in the diagram.""" if gr.graph.has_key('type'): topo_type = gr.graph['type'] # e.g. 'IS-IS' else: topo_type = 'unknown' if gr.graph.has_key('region'): topo_region = gr.graph['region'] # e.g. 'EU' else: topo_region = 'unknown' if gr.graph.has_key('timestamp'): topo_time = time.gmtime(gr.graph['timestamp']) topo_time = time.strftime("%Y-%m-%d_%H-%M-%S", topo_time) + "_GMT" else: topo_time = "unknown" font_labels = 'Helvetica' colors = {} colors['main'] = 'lightgray' # root window background colors['borders'] = 'lightslategray' # borders around nodes+clusters colors['pop'] = 'beige' # cluster/POP background colors['node'] = 'linen' # regular node fill color # List of normal link colors (cycle through all these colors to vary # link colors for ease of tracing individual links) colors['links'] = ['midnightblue', 'black', 'saddlebrown', 'purple'] # We really want the next colors to stand out, to show interesting things: colors['spof'] = 'lightsalmon' # node is a single point of failure colors['isolated'] = 'yellow' # node would be isolated by SPOF failure colors['removed'] = 'red' # element has been removed since previous topo colors['added'] = '#01DF01' # (green) element has been added since previous topo # # If we got a topology diff, we need to re-add the nodes+edges that are now # missing since the previous capture, or they won't be drawn in the current # topology diagram. if changes.has_key('nodes_removed'): for node in changes['nodes_removed']: gr.add_node(node) if changes.has_key('edges_removed'): for edge in changes['edges_removed']: gr.add_edge(edge[0], edge[1], weight=1) # # Use PyDot/GraphViz to draw the topology. The variable options here # (e.g. nodesep) are specified in the Dot language. dotgraph = pydot.Dot( "IS-IS Topology", graph_type='graph', bgcolor = colors['main'], nodesep = '0.25', ratio = '0.25', # 'ortho': edges horiz+vert w/ sharp corners, as in a circuit schematic # splines = 'ortho', rankdir = 'UD', # UD=up/down, LR=left/right center = 'true', remincross = 'true', # simplify: draw only one edge between connected nodes simplify = 'true') # legend = pydot.Cluster( "Legend", label = "LEGEND\n" +\ "Topology: " + topo_type +"\n" +\ "Region: " + topo_region + "\n" +\ "Recorded: " + topo_time + " \n\n\ Note: Except for red (removed) and green (added),\n\ link colors vary for ease of traceability.\n\n\ (SPOF: Single Point of Failure)\n\n", fontsize = 16, parent_graph = dotgraph, style = 'filled', color = colors['borders'], fillcolor = "white") if impacts != {}: legend.add_node(pydot.Node( "Potential\nIsolation", style = 'filled', fillcolor = colors['isolated'])) legend.add_node(pydot.Node( "Potential\nSPOF", style = 'filled', fillcolor = colors['spof'])) if changes != {}: legend.add_node(pydot.Node( "Added", style = 'filled', fillcolor = colors['added'])) legend.add_node(pydot.Node( "Removed\n(Failed?)", style = 'filled', fillcolor = colors['removed'])) legend.add_node(pydot.Node( "Router", style = 'filled', fillcolor = colors['node'])) dotgraph.add_subgraph(legend) # for node in gr.nodes(): # Assuming the first 3 char's of the node name is the POP/location cluster_name = node[0:3] # create subgraph for the POP if it's not there already if dotgraph.get_subgraph(cluster_name) == []: subgr = pydot.Cluster( cluster_name, parent_graph = dotgraph, style = 'rounded', label = cluster_name, font = font_labels, fontsize = '16', color = colors['borders'], fillcolor = colors['pop']) if changes.has_key('nodes_removed') and node in changes['nodes_removed']: nodefill = colors['removed'] elif impacts.has_key('spofs') and node in impacts['spofs']: nodefill = colors['spof'] elif impacts.has_key('isolated') and node in impacts['isolated']: nodefill = colors['isolated'] elif changes.has_key('nodes_added') and node in changes['nodes_added']: nodefill = colors['added'] else: nodefill = colors['node'] subgr.add_node(pydot.Node( node, URL = url_node(node), font = font_labels, fontsize = '10', style = "filled", color = colors['borders'], fillcolor = nodefill)) dotgraph.add_subgraph(subgr) xcolor = 0 # link color selector; link colors vary for ease of tracing for edge in gr.edges(): if changes.has_key('edges_removed') and edge in changes['edges_removed']: ecolor = colors['removed'] elif changes.has_key('edges_added') and edge in changes['edges_added']: ecolor = colors['added'] else: ecolor = colors['links'][xcolor] dotedge = pydot.Edge(edge, color = ecolor) dotgraph.add_edge(dotedge) xcolor += 1 if xcolor >= len(colors['links']): xcolor = 0 # We could draw a JPG/GIF/PNG/etc, but: # Generating SVG/DOT/XDOT is faster, smaller output, and easier to # manipulate on-the-fly for interactivity (e.g. Raphael, canviz). topos = {} if return_svg: topos['svg'] = dotgraph.create(prog='dot', format='svg') if return_graphml: topos['graphml'] = '\n'.join(nx.generate_graphml(gr)) if save_svg: fname = "%s_%s_%s.svg" % (topo_time, topo_type, topo_region) dotgraph.write(fname, prog='dot', format='svg') if save_graphml: fname = "%s_%s_%s.graphml" % (topo_time, topo_type, topo_region) nx.write_graphml(gr, fname) return topos
g.node['moo']['voltage'] = 110 if (out != ''): nx.write_graphml(g, out) if (json): print("---------------LINK DATA------------------") d0 = json_graph.node_link_data(g) print(d0) print("------------ADJACENCY DATA----------------") d1 = json_graph.adjacency_data(g) print(d1) print("------------------------------------------") if (prGraph): s = '\n'.join(nx.generate_graphml(g)) print(s) if (plot or save != ''): try: import matplotlib.pyplot as plt except: print('matplotlib not found') exit(-1) nx.draw_networkx(g) plt.axis('off') if (save != ''): plt.savefig(save) if (plot): plt.show()
def graph2graphml(graph): return "\n".join(networkx.generate_graphml(graph)) + "\n"
def print_graphml(net): strg = '\n'.join(nx.generate_graphml(net)) print strg
def to_graphml(net): return '\n'.join(nx.generate_graphml(net))
def graph2graphml(graph): return '\n'.join(networkx.generate_graphml(graph)) + '\n'
def centrality(): start_time = datetime.now() # TODO add config file read # TODO support cross network calculations (author_node --is--> author_node) ## >Get the REQUIRED parameters req_params = {} for entry in req_param_list: if request.args.get(entry) is not None: req_params[entry] = urllib2.unquote(request.args.get(entry)).replace("'", "") else: ret_string = "Required parameter missing: " + entry return jsonify(result=ret_string) # TODO Validate start_date, end_date ## >Verify the metric is valid if req_params["metric"] not in metric_list: return jsonify(result="Invalid metric requested") ## >Verify the start date is before the end date if int(req_params["start_date"]) > int(req_params["end_date"]): return jsonify(result="End data before start date") ## >Get the OPTIONAL parameters opt_params = {} for entry in opt_param_list: if request.args.get(entry) is not None: opt_params[entry] = urllib2.unquote(request.args.get(entry)).replace("'", "") else: opt_params[entry] = None # TODO validate the optional parameters ## >Get the FORMAT parameters for_params = {} for entry in format_param_list: if request.args.get(entry) is not None: for_params[entry] = urllib2.unquote(request.args.get(entry)).replace("'", "") else: for_params[entry] = None params = dict(req_params.items() + opt_params.items() + for_params.items()) ## >Build the mongo query mongo_query = {} mongo_query["PostDate"] = {"$gte": params["start_date"], "$lte": params["end_date"]} mongo_query["Network"] = params["network"] for param, value in opt_params.iteritems(): if value is not None: if param is "type": mongo_query["Type"] = opt_params["type"] if param is "twit_collect": mongo_query["Meta.sources"] = {"$in": [opt_params["twit_collect"]]} if param is "matched_project": mongo_query["Matching"] = {"$elemMatch": {"ProjectId": opt_params["matched_project"]}} if param is "matched_topic": # TODO pass if param is "scored_project": # TODO pass if param is "scored_topic": # TODO pass ## >Check if there are any matches if author_collection.find(mongo_query).count == 0: return "No connections found matching the criteria" else: ## >Map/reduce the A-->A connections a2a_map = Code( """ function () { emit({"author": this.Author, "connection": this.Connection}, {"count": 1} ); } """ ) a2a_reduce = Code( """ function (key, values) { var count = 0; values.forEach(function(v) { count += v['count']; }); return {"count": count}; } """ ) a2a_result = author_collection.map_reduce(a2a_map, a2a_reduce, "a2a_results", query=mongo_query).find() ## >Build the author list author_list = [] for a2a_count in a2a_result: author_list.append( ( a2a_count["_id"]["author"].replace("&", "/x26"), a2a_count["_id"]["connection"].replace("&", "/x26"), int(a2a_count["value"]["count"]), ) ) ## >Influence Calculations if len(author_list) > 0: ## >Create a black graph G = nx.DiGraph() ## >Add the endges to the graph G.add_weighted_edges_from(author_list) ## >Run the requested metric, on the graph 'G' calc_metric, stats = inf.run_metric(params["metric"], G, "weight", True) else: return jsonify(result="Parameters produced no graph/metrics") ## >Build the dictionary to return data_results = {} ## >Append the metric data data_results["metrics"] = calc_metric ## >If graph requested if for_params["return_graph"] is not None: if for_params["return_graph"].lower() == "true": ## >If format = data if for_params["format"] is None: ## >Append the graph data data_results["graph"] = nx.to_edgelist(G, nodelist=None) ## >If format = graphml elif for_params["format"].lower() == "graphml": ## >Create the graphml filename graphml_name = inf_sup.create_filename(params) ## >Get the graphml data graphml_data = "\n".join(nx.generate_graphml(G)) ## >Add the versioning graphml_final = '<?xml version="1.0" encoding="UTF-8"?>' + "\n" h = HTMLParser.HTMLParser() for line in graphml_data.split("\n"): ## >Escape the html content line = h.unescape(line) ## >For each node add appropriate metric data into the graphml if '<node id="' in line: graphml_final += line.replace("/>", ">") + "\n" node_name = line.partition('"')[-1].rpartition('"')[0] graphml_final += ' <data key="d1">' + str(calc_metric[node_name]) + "</data>" + "\n" graphml_final += " </node>" + "\n" else: graphml_final += line + "\n" ## >Add the key for the metric attribute if "<key" in line: graphml_final += ( ' <key attr.name="' + params["metric"] + '" attr.type="float" for="node" id="d1" />' ) if app.debug is True: ## >Write out the graphml for testing with open(graphml_name, "w") as output_file: for line in graphml_final: output_file.write(line.encode("utf-8")) if not output_file.closed: output_file.close() ## >Create the appropriate response to return the graphml response = make_response(graphml_final) response.headers["Content-Type"] = "text/xml" response.headers["Content-Distribution"] = "attachment; filename=%s" % (graphml_name,) return response if app.debug is True: ## >If debug mode add the query parameters data_results["query"] = params ## >And add statistics about the process statistics = {} statistics["runtime"] = str(datetime.now() - start_time) data_results["stats"] = statistics ## >Add the mongo query used data_results["query"] = mongo_query return jsonify(result=data_results)
def create_graphml(self, object): """"Write graphml with networkx.""" graphml = '\n'.join(nx.generate_graphml(object)) return graphml
def restore(dataset, timestamp=None, fmt='dot'): """Restore reconstructs the network topology at a specific time in the past. Restore replays gossip messages from a dataset and reconstructs the network as it would have looked like at the specified timestamp in the past. The network is then printed to stdout using the format specified with `--fmt`. """ if timestamp is None: timestamp = time.time() cutoff = timestamp - 2 * 7 * 24 * 3600 channels = {} nodes = {} # Some target formats do not suport UTF-8 aliases. codec = 'UTF-8' if fmt in ['dot'] else 'ASCII' for m in tqdm(dataset, desc="Replaying gossip messages"): if isinstance(m, ChannelAnnouncement): channels[f"{m.short_channel_id}/0"] = { "source": m.node_ids[0].hex(), "destination": m.node_ids[1].hex(), "timestamp": 0, "features": m.features.hex(), } channels[f"{m.short_channel_id}/1"] = { "source": m.node_ids[1].hex(), "destination": m.node_ids[0].hex(), "timestamp": 0, "features": m.features.hex(), } elif isinstance(m, ChannelUpdate): scid = f"{m.short_channel_id}/{m.direction}" chan = channels.get(scid, None) ts = m.timestamp if ts > timestamp: # Skip this update, it's in the future. continue if ts < cutoff: # Skip updates that cannot possibly keep this channel alive continue if chan is None: raise ValueError( f"Could not find channel with short_channel_id {scid}") if chan["timestamp"] > ts: # Skip this update, it's outdated. continue chan["timestamp"] = ts chan["fee_base_msat"] = m.fee_base_msat chan["fee_proportional_millionths"] = m.fee_proportional_millionths chan["htlc_minimim_msat"] = m.htlc_minimum_msat if m.htlc_maximum_msat: chan["htlc_maximum_msat"] = m.htlc_maximum_msat chan["cltv_expiry_delta"] = m.cltv_expiry_delta elif isinstance(m, NodeAnnouncement): node_id = m.node_id.hex() old = nodes.get(node_id, None) if old is not None and old["timestamp"] > m.timestamp: continue alias = m.alias.replace(b'\x00', b'').decode(codec, 'ignore') nodes[node_id] = { "id": node_id, "timestamp": m.timestamp, "features": m.features.hex(), "rgb_color": m.rgb_color.hex(), "alias": alias, "addresses": ",".join([str(a) for a in m.addresses]), "out_degree": 0, "in_degree": 0, } # Cleanup pass: drop channels that haven't seen an update in 2 weeks todelete = [] for scid, chan in tqdm(channels.items(), desc="Pruning outdated channels"): if chan["timestamp"] < cutoff: todelete.append(scid) else: node = nodes.get(chan["source"], None) if node is None: continue else: node["out_degree"] += 1 node = nodes.get(chan["destination"], None) if node is None: continue else: node["in_degree"] += 1 for scid in todelete: del channels[scid] nodes = [ n for n in nodes.values() if n["in_degree"] > 0 or n['out_degree'] > 0 ] if len(channels) == 0: print( "ERROR: no channels are left after pruning, make sure to select a" "timestamp that is covered by the dataset.") sys.exit(1) g = nx.DiGraph() for n in nodes: g.add_node(n["id"], **n) for scid, c in channels.items(): g.add_edge(c["source"], c["destination"], scid=scid, **c) if fmt == 'dot': print(nx.nx_pydot.to_pydot(g)) elif fmt == 'gml': for line in nx.generate_gml(g): print(line) elif fmt == 'graphml': for line in nx.generate_graphml(g, named_key_ids=True, edge_id_from_attribute='scid'): print(line) elif fmt == 'json': print(json.dumps(json_graph.adjacency_data(g)))
def addwf(request): fileinfo = None jobinfo = None sticky = False inherit = None post = request.POST dagName = post['dag'] name = post['name'] user = post['user'] state = post['state'] filejson = post['fileinfo'] jobjson = post['jobinfo'] description = post['description'] wfuuid = uuid.uuid1() if (filejson != ''): try: fileinfo = json.loads(filejson) logger.info('WORKFLOW: fileinfo %s', fileinfo) except: # non-JSON string... Identify: if (filejson == 'sticky'): sticky = True logger.info('WORKFLOW: sticky filename set') if ('inherit' in filejson): try: inherit = filejson.split(':')[1] logger.info('WORKFLOW: filename inheritance %s', inherit) except: pass if (jobjson != ''): jobinfo = json.loads(jobjson) # print(jobjson) # print(jobinfo) ts_def = timezone.now() myDAG = dag.objects.get(name=dagName) rootName = myDAG.root nvert = myDAG.nvertices # Create a Workflow object and populate it: wf = workflow() wf.ts_def = ts_def wf.uuid = wfuuid wf.dag = dagName wf.name = name wf.user = user wf.state = state wf.nvertices = nvert wf.description = description # ATTN: we'll save the WF a bit later ( we would like to get the root job uuid). g = nx.DiGraph() # +++++++++ JOBS for dv in dagVertex.objects.filter(dag=dagName): g.add_node(dv.name, wf=dagName) # defaults: payload = dv.payload env = dv.env if (jobinfo): # need to overwrite some attributes for k in jobinfo.keys(): if (k == dv.name): try: payload = jobinfo[k]["payload"] except: pass try: env = json.dumps(jobinfo[k]["env"]) except: pass j = job( uuid=uuid.uuid1(), wfuuid=wfuuid, jobtype=dv.jobtype, payload=payload, env=env, priority=dv.priority, state='template', ts_def=ts_def, timelimit=dv.timelimit, name=dv.name, ) if (dv.name == rootName): wf.rootuuid = j.uuid j.save() # +++++++++ END JOBS wf.save() # can do it now since needed rootuuid, now we have it # +++++++++ DATA for de in dagEdge.objects.filter(dag=dagName): # even in a multigraph, an edge can only be associated with one # source and one target. A source and a target can be connected # by multiple edges in a multigraph, however. So check for # the former condition. s_cand = job.objects.filter(wfuuid=wfuuid, name=de.source) t_cand = job.objects.filter(wfuuid=wfuuid, name=de.target) if (len(s_cand) != 1): return HttpResponse("addwf: inconsistent graph - source") if (len(t_cand) != 1): return HttpResponse("addwf: inconsistent graph - target") sourceuuid = s_cand[0].uuid targetuuid = t_cand[0].uuid # a "dataset" - a file in this case - is created by default with a name formed # from its UUID and predefined extension (as per type object) # It can, however, be overridden by supplemental information foubd in "fileinfo" # Defaults: d_uuid = str(uuid.uuid1()) dt = '' try: dt = datatype.objects.get(name=de.datatype) except: delstat = '' if delwf(wfuuid): delstat = 'OK' else: delstat = 'FAIL' return HttpResponse( 'Failed to get datatype, deleting workflow. Clean up status: %s' % delstat) ext = dt.ext # expect that the dot is included dirpath = de.dirpath filename = '' if (sticky): filename = de.name elif (inherit): filename = inherit + ':' + dagName + ':' + de.source + ':' + de.datatag + ext else: filename = d_uuid + ext comment = de.comment # default comment inherited from DAG # Optional overrides: client may send JSON data with overriding data if (fileinfo): for k in fileinfo.keys(): if ((de.source + ":" + de.target) == k): try: filename = fileinfo[k]["name"] except: pass try: dirpath = fileinfo[k]["dirpath"] except: pass try: comment = fileinfo[k]["comment"] except: pass logger.info('WORKFLOW: filename %s', filename) d = dataset( uuid=d_uuid, wfuuid=wfuuid, source=de.source, target=de.target, sourceuuid=sourceuuid, targetuuid=targetuuid, name=filename, state='template', dirpath=dirpath, comment=comment, datatype=de.datatype, datatag=de.datatag, wf=name, ) d.save() g.add_edge(de.source, de.target) # +++ AUGMENT JOB ENVIRONMENT WITH DATASET INFO fullname = dirpath + filename #print('source:',de.source,' target:', de.target,' datatag:', de.datatag, ' full name:',fullname) for jid in (sourceuuid, targetuuid): j4env = job.objects.get( uuid=jid) # MUST work as per comments above, we checked j4env.augmentEnv({de.datatag: fullname }) # Note that the filename is set in the env.. j4env.save() # +++++++++ END DATA # Now that the WF is created, is can be set into defined state # (hence ready for execution) if so desired. This method # toggles both wf and its "root job" if (state == 'defined'): wf2defined(wf) # Finish up and send message to the client s = '\n'.join(nx.generate_graphml(g)) # this can be in fact something else return HttpResponse(s)
def to_graphml(g): return generate_graphml(g)