def display_graph_from_list(t, l=None):
if l == None:
return
for element in l:
if element["_type"] == "vertex":
node_temp = Node(element["_id"])
for key in element:
node_temp.property[key] = element[key]
node_temp.property["label"] = node_temp.property["name"]
node_temp.property["colour"] = node_temp.property["color"]
t.add_node(node_temp)
elif element["_type"] == "edge":
src = element["_outV"]
dst = element["_inV"]
edge_temp = Edge(src, dst, directed=True)
for key in element:
edge_temp.property[key] = element[key]
t.add_edge(edge_temp)
t.commit()
def display_graph(t, nodes, edges):
if nodes != None:
print "*** Graphing Nodes"
for n in nodes:
node_temp = Node(n._id)
properties = n.map()
for key in properties:
node_temp.property[key] = properties[key]
node_temp.property["colour"] = node_temp.property["color"]
node_temp.property["label"] = node_temp.property["name"]
t.add_node(node_temp)
if edges != None:
print "*** Graphing Edges"
for e in edges:
src = e._outV
dst = e._inV
edge_temp = Edge(src, dst, directed=True)
properties = e.map()
for key in properties:
edge_temp.property[key] = properties[key]
t.add_edge(edge_temp)
t.commit()
def display_graph_from_list(t, l=None):
if l == None:
return
for element in l:
if element["_type"] == "vertex":
node_temp = Node(element["_id"])
for key in element:
node_temp.property[key] = element[key]
node_temp.property["label"] = node_temp.property["name"]
node_temp.property["colour"] = node_temp.property["color"]
t.add_node(node_temp)
elif element["_type"] == "edge":
src = element["_outV"]
dst = element["_inV"]
edge_temp = Edge(src, dst, directed=True)
for key in element:
edge_temp.property[key] = element[key]
t.add_edge(edge_temp)
t.commit()
def display_graph(t, nodes, edges):
if nodes != None:
print "*** Graphing Nodes"
for n in nodes:
node_temp = Node(n._id)
properties = n.map()
for key in properties:
node_temp.property[key] = properties[key]
node_temp.property["colour"] = node_temp.property["color"]
node_temp.property["label"] = node_temp.property["name"]
t.add_node(node_temp)
if edges != None:
print "*** Graphing Edges"
for e in edges:
src = e._outV
dst = e._inV
edge_temp = Edge(src, dst, directed=True)
properties = e.map()
for key in properties:
edge_temp.property[key] = properties[key]
t.add_edge(edge_temp)
t.commit()
def set_edge(self, node1_id, node2_id, realtime=True, edge_attrs={}):
'''Add a an edge to the visualized graph. Nodes that do not
exist in the current graph data structure (i.e. NetworkX graph)
will be automatically created.
If optional parameter <vis_realtime> is True, gephi visualization will
be automatically updated.
"If you define twice a node, it won't do anything.
If you define an edge with a node that doesn't exist, it won't do anything.
If you update a node that doesn't exist, it won't do anything.
-Koumin"
'''
if self.G.has_edge(node1_id, node2_id) and edge_attrs:
self.update_edge_data(node1_id, node2_id, edge_attrs=edge_attrs)
else:
self.G.add_edge(node1_id, node2_id, edge_attrs)
# add node to graphif it does no exist in graph
n1_vis = self.set_node(node1_id, realtime=False)
n2_vis = self.set_node(node2_id, realtime=False)
# Edge(source, target, is_directed)
new_edge = Edge(n1_vis, n2_vis, False, **GraphVis.default_edge_vis)
self.add_edge(new_edge)
# update visualization edge properties
new_edge.property.update(edge_attrs)
self.change_edge(new_edge)
if realtime:
self.commit()
return new_edge
def process_item(self, item, spider):
patent_args = {'size': 5, 'red': 1, 'green': 0, 'blue': 0}
patent_node = Node(item['publication_number'], **patent_args)
patent_node.property['type'] = 'patent'
patent_node.property['title'] = item.get('title')
patent_node.property['filing_date'] = item.get('filing_date')
patent_node.property['publication_date'] = item.get('publication_date')
patent_node.property['priority_date'] = item.get('priority_date')
patent_node.property['grant_date'] = item.get('grant_date')
patent_node.property['pdf'] = item.get('pdf')
if item['publication_number'] in self.nodes:
self.gephi.change_node(patent_node)
else:
self.gephi.add_node(patent_node)
link_args = {'size': 5, 'red': 0, 'green': 0, 'blue': 1}
for citation in item.get('citations', []):
citation_node = Node(citation, **link_args)
citation_node.property['type'] = 'link'
self.gephi.add_node(citation_node)
self.gephi.add_edge(Edge(patent_node, citation_node, True))
self.nodes.add(citation)
for cited_by in item.get('cited_by', []):
cited_by_node = Node(cited_by, **link_args)
cited_by_node.property['type'] = 'link'
self.gephi.add_node(cited_by_node)
self.gephi.add_edge(Edge(cited_by_node, patent_node, True))
self.nodes.add(cited_by)
entity_args = {'size': 5, 'red': 0, 'green': 1, 'blue': 0}
entities = set(item.get('inventors', []) + item.get('assignees', []))
for entity in entities:
entity_node = Node(entity, **entity_args)
entity_node.property['type'] = 'entity'
self.gephi.add_node(entity_node)
self.gephi.add_edge(Edge(entity_node, patent_node, True))
self.logger.info('Publishing item {}'.format(item['publication_number']))
try:
self.gephi.commit()
except ConnectionError, e:
self.logger.error(e)
def received_message(self, m):
#Loading the data as json
data = json.loads("%s"%m)
print "==%s=="%data['x']['hash']
#Created the node that represent the transaction
transactionNode = Node(data['x']['hash'],blue=1)
#With some properties
for prop in ['vin_sz','vout_sz','lock_time','relayed_by','tx_index','time']:
transactionNode.property[prop]=data['x'][prop]
#Hack to avoid "size" of the node
transactionNode.property['transaction_size'] = data['x']['size']
#we type our node
transactionNode.property['type']='Transaction'
t.add_node(transactionNode)
#For all incomming flow
for inp in data['x']['inputs']:
print inp['prev_out']['addr']
#Create a Node with properties
inNode = Node(inp['prev_out']['addr'],red=1)
inNode.property['type']='Wallet'
inNode.property['time']=data['x']["time"]
t.add_node(inNode)
#Create an edge Wallet-[weight=value of the transaction]->Transaction
edge = Edge(inNode,transactionNode,True,weight=inp['prev_out']['value'])
edge.property['type'] = inp['prev_out']['type']
t.add_edge(edge)
print "*"
#For all outgoing flow
for out in data['x']['out'] :
print out['addr']
#Create a Node with properties
outNode = Node(out['addr'],red=1)
outNode.property['type']='Wallet'
outNode.property['time']=data['x']["time"]
t.add_node(outNode)
#Create an edge Transaction-[weight=value of the transaction]->Wallet
edge = Edge(transactionNode,outNode,True,weight=out['value'])
edge.property['type'] = out['type']
t.add_edge(edge)
t.commit()
def received_message(self, m):
print "===="
inNode = []
outNode = []
data = json.loads("%s" % m)
#Get All in Nodes of the transaction
for inp in data['x']['inputs']:
print inp['prev_out']['addr']
inNode += [Node(inp['prev_out']['addr'])]
print "*"
#Get All out Nodes of the transaction
for out in data['x']['out']:
print out['addr']
outNode += [Node(out['addr'])]
#Graph All the Things !
for n in inNode:
t.add_node(n)
for o in outNode:
t.add_node(o)
t.add_edge(Edge(n, o, True))
t.commit()
def send_graph(G, gs, rgb=(1,0,0), label_fcn = None):
'''
Sends the networkX graph, G, to gephi as a JSON stream
:param: G graph to send
:param: gs GraphVis instance
:param: rgb node color
:param: id_fcn function to generate node label
'''
gnodes = make_gephi_nodes(G.nodes(), rgb=rgb, size=10)
for k,v1 in gnodes.items():
gs.add_node(v1)
try:
for v2 in make_gephi_nodes(G[k].keys()).values(): # G[k] is networkx node neighbor dictionary
gs.add_node(v2)
gs.add_edge(Edge(v1, v2, False, weight=1))
gs.commit()
except Exception, e:
print "Unable to add", k
print e
def received_message(self, m):
#Loading the data as json
data = json.loads("%s" % m)
print "==%s==" % data['x']['hash']
#Created the node that represent the transaction
transactionNode = Node(data['x']['hash'], blue=1)
#With some properties
for prop in [
'vin_sz', 'vout_sz', 'lock_time', 'relayed_by', 'tx_index',
'time'
]:
transactionNode.property[prop] = data['x'][prop]
#Hack to avoid "size" of the node
transactionNode.property['transaction_size'] = data['x']['size']
#we type our node
transactionNode.property['type'] = 'Transaction'
t.add_node(transactionNode)
#For all incomming flow
for inp in data['x']['inputs']:
print inp['prev_out']['addr']
#Create a Node with properties
inNode = Node(inp['prev_out']['addr'], red=1)
inNode.property['type'] = 'Wallet'
inNode.property['time'] = data['x']["time"]
t.add_node(inNode)
#Create an edge Wallet-[weight=value of the transaction]->Transaction
edge = Edge(inNode,
transactionNode,
True,
weight=inp['prev_out']['value'])
edge.property['type'] = inp['prev_out']['type']
t.add_edge(edge)
print "*"
#For all outgoing flow
for out in data['x']['out']:
print out['addr']
#Create a Node with properties
outNode = Node(out['addr'], red=1)
outNode.property['type'] = 'Wallet'
outNode.property['time'] = data['x']["time"]
t.add_node(outNode)
#Create an edge Transaction-[weight=value of the transaction]->Wallet
edge = Edge(transactionNode, outNode, True, weight=out['value'])
edge.property['type'] = out['type']
t.add_edge(edge)
t.commit()
