def worker(id, queue, history): web_sock = streamer.GephiWS(workspace='workspace1') stream = streamer.Streamer(web_sock) def add_to_graph(origin, linked_articles): o_node = graph.Node(origin.id, label=origin.title, namespace=origin.namespace) stream.change_node(o_node) for article in linked_articles: node = graph.Node(article.id, label=article.title, namespace=article.namespace) stream.add_node(node) edge = graph.Edge(o_node, node) stream.add_edge(edge) stream.commit() while True: try: article = queue.pop() except KeyError: continue print('[{}] Working on: {}'.format(id, article)) history.append(article.id) article.scrape() for linked in article.linked_articles: if linked.id in history: continue if len(queue) < 500: # limit queue size queue.add(linked) add_to_graph(article, article.linked_articles)
def gephi_start(self, threshold=28, workspace='chemspace'): self.stream = streamer.Streamer(streamer.GephiWS(workspace=workspace)) base = self.fingerprints_df[self.base] distances = [[ ChemicalSpaceGraph.similarity(self.fingerprints_df[x], base), x ] for x in self.nodes] distances.sort() rankings = {x[1]: [i, x[0]] for i, x in enumerate(distances)} nodes = [ graph.Node(x, distance=rankings[x][1], ranking=rankings[x][0]) for x in self.nodes ] self.stream.add_node(*nodes) similarities = pd.Series(list(self.edges.values())) m = similarities.max() edges = [ graph.Edge(x, y, directed=False, weight=1 - self.edges[(x, y)] / m, label=self.edges[(x, y)]) for x, y in self.edges if self.edges[(x, y)] <= threshold ] self.stream.add_edge(*edges)
def export_gephi(self): try: tmp_host = self.sel_host.split("_") print(tmp_host[0]) stream = streamer.Streamer( streamer.GephiWS(hostname="localhost", port=8080, workspace="workspace1")) for domain in self.obj_dns_parser.csv_obj.h.h.hosts[ tmp_host[0]].domain: node_b = graph.Node(domain, size=1) for txn_id in self.obj_dns_parser.csv_obj.h.h.hosts[ tmp_host[0]].domain[domain].list: for resp in self.obj_dns_parser.csv_obj.h.h.hosts[ tmp_host[0]].domain[domain].list[txn_id].response: node_a = graph.Node(resp.resolved_ip, size=2) stream.add_node(node_a) stream.add_node(node_b) edge_ab = graph.Edge(node_a, node_b) stream.add_edge(edge_ab) except: self.lbl_status.config(text="Failed in export_gephi.")
def main(): streamer.GephiWS(hostname='localhost', port=8080, workspace='workspace0') stream = streamer.Streamer(streamer.GephiWS()) node_a = graph.Node("A") node_b = graph.Node("B") stream.add_node(node_a, node_b) edge = graph.Edge(node_a, node_b) stream.add_edge(edge)
def __init__(self, hostname, port, workspace): self.stream = streamer.Streamer( streamer.GephiWS(hostname="192.168.10.108", port=8080, workspace="workspace1")) self.node_size = 200 # this scales up everything self.size_increment = 50 self.r_seed = 3000 self.nodedict = {}
def scraper(queue, history): web_sock = streamer.GephiWS(workspace='workspace1') stream = streamer.Streamer(web_sock) def add_to_graph(origin, related): o_node = graph.Node(origin.id, label=origin.name, url=origin.url, subscriptions=origin.subscriptions, views=origin.views) stream.change_node(o_node) for ch in related: node = graph.Node(ch.id, label=ch.name, url=ch.url, subscriptions=0, views=0) stream.add_node(node) edge = graph.Edge(o_node, node) stream.add_edge(edge) stream.commit() while True: try: channel = queue.pop() except KeyError: continue print('Working on: {}'.format(channel)) history.append(channel.id) channel.scrape() for ch in channel.related_channels: if ch.id in history: continue if len(queue) < 500: # limit queue size queue.add(ch) add_to_graph(channel, channel.related_channels)
def __init__(self, streamer=streamer.Streamer(streamer.GephiREST())): self.streamer = streamer self._bucket = {}
from gephistreamer import graph from gephistreamer import streamer stream = streamer.Streamer(streamer.GephiWS()) # Create a node with a custom_property node_a = graph.Node("A", custom_property=1) # Create a node and then add the custom_property node_b = graph.Node("B") node_b.property['custom_property'] = 2 # Add the node to the stream # you can also do it one by one or via a list # l = [node_a,node_b] # stream.add_node(*l) stream.add_node(node_a, node_b) # Create edge # You can also use the id of the node : graph.Edge("A","B",custom_property="hello") edge_ab = graph.Edge(node_a, node_b, custom_property="hello") stream.add_edge(edge_ab)
from gephistreamer import graph from gephistreamer import streamer import re import os import sys directory = "tasks/" fileType = 'transE' stream = streamer.Streamer(streamer.GephiREST()) #adds the nodes and edges from the all_data file def addFolder(path): try: abspath = os.path.join(path, 'all_data') if os.path.isfile(abspath): print abspath with open(abspath, 'r') as f: contents = f.read().splitlines() for line in contents: #get first columb data items = re.split(r'\t+', line) labels, labels_1, labels_2 = re.split( r':+', items[0]), re.split(r':+', items[1]), re.split(r':+', items[2]) #type = labels[1] stream.add_node(graph.Node(labels[2], type=labels[1]), graph.Node(labels_1[2], type=labels_1[1]))
if type(text) != str: return iter_text = iter(text.split('/r/')) next(iter_text) for text in iter_text: that_sub = re.split('[^a-zA-Z0-9_]', text)[0].lower() if that_sub == "" or this_sub == that_sub: continue subreddit_edges[tuple(sorted((this_sub, that_sub)))] += 1 for this_subreddit, row in data.iterrows(): process_text(row['description'], this_subreddit) process_text(row['submit_text'], this_subreddit) stream = streamer.Streamer(streamer.GephiWS(workspace="workspace1")) def add_node(subreddit): try: size = data.loc[subreddit].normed_subscribers category = data.loc[subreddit].category nsfw = data.loc[subreddit].over_18 except KeyError: size = 1 category = "" nsfw = False if nsfw: red, green, blue = 0, 0, 1 else:
from ws4py.client.threadedclient import WebSocketClient from gephistreamer import graph from gephistreamer import streamer import json import random t = streamer.Streamer(streamer.GephiREST(port=8080)) class DummyClient(WebSocketClient): #Bootstrap to register to websocket def opened(self): self.send('{"op":"unconfirmed_sub"}') def closed(self, code, reason=None): print("Closed down", code, reason) #When we receive a new blockchain def received_message(self, m): #Loading the data as json data = json.loads(str(m)) print(json.dumps(data, sort_keys=True, indent=4)) #Created the node that represent the transaction transactionNode = graph.Node(data['x']['hash'], blue=1, x=random.randint(0, 500), y=random.randint(0, 500)) #With some properties for prop in [ 'vin_sz', 'vout_sz', 'lock_time', 'relayed_by', 'tx_index',
# -*- coding: utf-8 -*- """ Created on Sat Jul 14 06:52:48 2018 @author: Administrator """ from gephistreamer import graph from gephistreamer import streamer # Create a Streamer # adapt if needed : streamer.GephiWS(hostname="localhost", port=8080, workspace="workspace0") # You can also use REST call with GephiREST (a little bit slower than Websocket) GWS = streamer.GephiWS(hostname="localhost", port=8080, workspace="workspace1") stream = streamer.Streamer(GWS) # Create a node with a custom_property node_a = graph.Node("A", custom_property=1) # Create a node and then add the custom_property node_b = graph.Node("B") node_b.property['custom_property'] = 2 # Add the node to the stream # you can also do it one by one or via a list l = [node_a, node_b] stream.add_node(*l) node_c = graph.Node("C") stream.add_node(node_a, node_b, node_c) # Create edge