def worker(id, queue, history):
web_sock = streamer.GephiWS(workspace='workspace1')
stream = streamer.Streamer(web_sock)
def add_to_graph(origin, linked_articles):
o_node = graph.Node(origin.id, label=origin.title, namespace=origin.namespace)
stream.change_node(o_node)
for article in linked_articles:
node = graph.Node(article.id, label=article.title, namespace=article.namespace)
stream.add_node(node)
edge = graph.Edge(o_node, node)
stream.add_edge(edge)
stream.commit()
while True:
try:
article = queue.pop()
except KeyError:
continue
print('[{}] Working on: {}'.format(id, article))
history.append(article.id)
article.scrape()
for linked in article.linked_articles:
if linked.id in history:
continue
if len(queue) < 500: # limit queue size
queue.add(linked)
add_to_graph(article, article.linked_articles)
def gephi_start(self, threshold=28, workspace='chemspace'):
self.stream = streamer.Streamer(streamer.GephiWS(workspace=workspace))
base = self.fingerprints_df[self.base]
distances = [[
ChemicalSpaceGraph.similarity(self.fingerprints_df[x], base), x
] for x in self.nodes]
distances.sort()
rankings = {x[1]: [i, x[0]] for i, x in enumerate(distances)}
nodes = [
graph.Node(x, distance=rankings[x][1], ranking=rankings[x][0])
for x in self.nodes
]
self.stream.add_node(*nodes)
similarities = pd.Series(list(self.edges.values()))
m = similarities.max()
edges = [
graph.Edge(x,
y,
directed=False,
weight=1 - self.edges[(x, y)] / m,
label=self.edges[(x, y)]) for x, y in self.edges
if self.edges[(x, y)] <= threshold
]
self.stream.add_edge(*edges)
def export_gephi(self):
try:
tmp_host = self.sel_host.split("_")
print(tmp_host[0])
stream = streamer.Streamer(
streamer.GephiWS(hostname="localhost",
port=8080,
workspace="workspace1"))
for domain in self.obj_dns_parser.csv_obj.h.h.hosts[
tmp_host[0]].domain:
node_b = graph.Node(domain, size=1)
for txn_id in self.obj_dns_parser.csv_obj.h.h.hosts[
tmp_host[0]].domain[domain].list:
for resp in self.obj_dns_parser.csv_obj.h.h.hosts[
tmp_host[0]].domain[domain].list[txn_id].response:
node_a = graph.Node(resp.resolved_ip, size=2)
stream.add_node(node_a)
stream.add_node(node_b)
edge_ab = graph.Edge(node_a, node_b)
stream.add_edge(edge_ab)
except:
self.lbl_status.config(text="Failed in export_gephi.")
def main():
streamer.GephiWS(hostname='localhost', port=8080, workspace='workspace0')
stream = streamer.Streamer(streamer.GephiWS())
node_a = graph.Node("A")
node_b = graph.Node("B")
stream.add_node(node_a, node_b)
edge = graph.Edge(node_a, node_b)
stream.add_edge(edge)
def __init__(self, hostname, port, workspace):
self.stream = streamer.Streamer(
streamer.GephiWS(hostname="192.168.10.108",
port=8080,
workspace="workspace1"))
self.node_size = 200 # this scales up everything
self.size_increment = 50
self.r_seed = 3000
self.nodedict = {}
def scraper(queue, history):
web_sock = streamer.GephiWS(workspace='workspace1')
stream = streamer.Streamer(web_sock)
def add_to_graph(origin, related):
o_node = graph.Node(origin.id,
label=origin.name,
url=origin.url,
subscriptions=origin.subscriptions,
views=origin.views)
stream.change_node(o_node)
for ch in related:
node = graph.Node(ch.id,
label=ch.name,
url=ch.url,
subscriptions=0,
views=0)
stream.add_node(node)
edge = graph.Edge(o_node, node)
stream.add_edge(edge)
stream.commit()
while True:
try:
channel = queue.pop()
except KeyError:
continue
print('Working on: {}'.format(channel))
history.append(channel.id)
channel.scrape()
for ch in channel.related_channels:
if ch.id in history:
continue
if len(queue) < 500: # limit queue size
queue.add(ch)
add_to_graph(channel, channel.related_channels)
def __init__(self, streamer=streamer.Streamer(streamer.GephiREST())):
self.streamer = streamer
self._bucket = {}
from gephistreamer import graph
from gephistreamer import streamer
stream = streamer.Streamer(streamer.GephiWS())
# Create a node with a custom_property
node_a = graph.Node("A", custom_property=1)
# Create a node and then add the custom_property
node_b = graph.Node("B")
node_b.property['custom_property'] = 2
# Add the node to the stream
# you can also do it one by one or via a list
# l = [node_a,node_b]
# stream.add_node(*l)
stream.add_node(node_a, node_b)
# Create edge
# You can also use the id of the node : graph.Edge("A","B",custom_property="hello")
edge_ab = graph.Edge(node_a, node_b, custom_property="hello")
stream.add_edge(edge_ab)
from gephistreamer import graph
from gephistreamer import streamer
import re
import os
import sys
directory = "tasks/"
fileType = 'transE'
stream = streamer.Streamer(streamer.GephiREST())
#adds the nodes and edges from the all_data file
def addFolder(path):
try:
abspath = os.path.join(path, 'all_data')
if os.path.isfile(abspath):
print abspath
with open(abspath, 'r') as f:
contents = f.read().splitlines()
for line in contents:
#get first columb data
items = re.split(r'\t+', line)
labels, labels_1, labels_2 = re.split(
r':+',
items[0]), re.split(r':+',
items[1]), re.split(r':+', items[2])
#type = labels[1]
stream.add_node(graph.Node(labels[2], type=labels[1]),
graph.Node(labels_1[2], type=labels_1[1]))
if type(text) != str:
return
iter_text = iter(text.split('/r/'))
next(iter_text)
for text in iter_text:
that_sub = re.split('[^a-zA-Z0-9_]', text)[0].lower()
if that_sub == "" or this_sub == that_sub:
continue
subreddit_edges[tuple(sorted((this_sub, that_sub)))] += 1
for this_subreddit, row in data.iterrows():
process_text(row['description'], this_subreddit)
process_text(row['submit_text'], this_subreddit)
stream = streamer.Streamer(streamer.GephiWS(workspace="workspace1"))
def add_node(subreddit):
try:
size = data.loc[subreddit].normed_subscribers
category = data.loc[subreddit].category
nsfw = data.loc[subreddit].over_18
except KeyError:
size = 1
category = ""
nsfw = False
if nsfw:
red, green, blue = 0, 0, 1
else:
from ws4py.client.threadedclient import WebSocketClient
from gephistreamer import graph
from gephistreamer import streamer
import json
import random
t = streamer.Streamer(streamer.GephiREST(port=8080))
class DummyClient(WebSocketClient):
#Bootstrap to register to websocket
def opened(self):
self.send('{"op":"unconfirmed_sub"}')
def closed(self, code, reason=None):
print("Closed down", code, reason)
#When we receive a new blockchain
def received_message(self, m):
#Loading the data as json
data = json.loads(str(m))
print(json.dumps(data, sort_keys=True, indent=4))
#Created the node that represent the transaction
transactionNode = graph.Node(data['x']['hash'],
blue=1,
x=random.randint(0, 500),
y=random.randint(0, 500))
#With some properties
for prop in [
'vin_sz', 'vout_sz', 'lock_time', 'relayed_by', 'tx_index',
# -*- coding: utf-8 -*-
"""
Created on Sat Jul 14 06:52:48 2018
@author: Administrator
"""
from gephistreamer import graph
from gephistreamer import streamer
# Create a Streamer
# adapt if needed : streamer.GephiWS(hostname="localhost", port=8080, workspace="workspace0")
# You can also use REST call with GephiREST (a little bit slower than Websocket)
GWS = streamer.GephiWS(hostname="localhost", port=8080, workspace="workspace1")
stream = streamer.Streamer(GWS)
# Create a node with a custom_property
node_a = graph.Node("A", custom_property=1)
# Create a node and then add the custom_property
node_b = graph.Node("B")
node_b.property['custom_property'] = 2
# Add the node to the stream
# you can also do it one by one or via a list
l = [node_a, node_b]
stream.add_node(*l)
node_c = graph.Node("C")
stream.add_node(node_a, node_b, node_c)
# Create edge