def display_graph(t, nodes, edges):
if nodes != None:
print "*** Graphing Nodes"
for n in nodes:
node_temp = Node(n._id)
properties = n.map()
for key in properties:
node_temp.property[key] = properties[key]
node_temp.property["colour"] = node_temp.property["color"]
node_temp.property["label"] = node_temp.property["name"]
t.add_node(node_temp)
if edges != None:
print "*** Graphing Edges"
for e in edges:
src = e._outV
dst = e._inV
edge_temp = Edge(src, dst, directed=True)
properties = e.map()
for key in properties:
edge_temp.property[key] = properties[key]
t.add_edge(edge_temp)
t.commit()
def display_graph_from_list(t, l=None):
if l == None:
return
for element in l:
if element["_type"] == "vertex":
node_temp = Node(element["_id"])
for key in element:
node_temp.property[key] = element[key]
node_temp.property["label"] = node_temp.property["name"]
node_temp.property["colour"] = node_temp.property["color"]
t.add_node(node_temp)
elif element["_type"] == "edge":
src = element["_outV"]
dst = element["_inV"]
edge_temp = Edge(src, dst, directed=True)
for key in element:
edge_temp.property[key] = element[key]
t.add_edge(edge_temp)
t.commit()
def display_graph_from_list(t, l=None):
if l == None:
return
for element in l:
if element["_type"] == "vertex":
node_temp = Node(element["_id"])
for key in element:
node_temp.property[key] = element[key]
node_temp.property["label"] = node_temp.property["name"]
node_temp.property["colour"] = node_temp.property["color"]
t.add_node(node_temp)
elif element["_type"] == "edge":
src = element["_outV"]
dst = element["_inV"]
edge_temp = Edge(src, dst, directed=True)
for key in element:
edge_temp.property[key] = element[key]
t.add_edge(edge_temp)
t.commit()
def display_graph(t, nodes, edges):
if nodes != None:
print "*** Graphing Nodes"
for n in nodes:
node_temp = Node(n._id)
properties = n.map()
for key in properties:
node_temp.property[key] = properties[key]
node_temp.property["colour"] = node_temp.property["color"]
node_temp.property["label"] = node_temp.property["name"]
t.add_node(node_temp)
if edges != None:
print "*** Graphing Edges"
for e in edges:
src = e._outV
dst = e._inV
edge_temp = Edge(src, dst, directed=True)
properties = e.map()
for key in properties:
edge_temp.property[key] = properties[key]
t.add_edge(edge_temp)
t.commit()
def set_node(self, node_id, realtime=True, node_attrs={}):
'''
Add a node to the network datastructure.
kwargs is an optional dictionary of node properties
If optional parameter <vis_realtime> is True, gephi visualization will be
automatically updated.'''
if self.G.has_node(node_id):
if node_attrs:
self.update_node_data(node_id, node_attrs)
else:
node_attrs=self.G.node[node_id]
else:
# note: here we add node to data graph (visual added below)
self.G.add_node(node_id, node_attrs)
# create GephiStreamer.Node instance to send
# typecast node_id to string
new_node = Node(str(node_id), **GraphVis.default_node_vis)
self.add_node(new_node)
# values from node_attrs override node_vis_props
# ['r', 'g', 'b'] for node colors except on init --> ['red', 'green', 'blue']
new_node.property.update(node_attrs)
self.change_node(new_node)
if realtime:
self.commit()
return new_node
def received_message(self, m):
#Loading the data as json
data = json.loads("%s"%m)
print "==%s=="%data['x']['hash']
#Created the node that represent the transaction
transactionNode = Node(data['x']['hash'],blue=1)
#With some properties
for prop in ['vin_sz','vout_sz','lock_time','relayed_by','tx_index','time']:
transactionNode.property[prop]=data['x'][prop]
#Hack to avoid "size" of the node
transactionNode.property['transaction_size'] = data['x']['size']
#we type our node
transactionNode.property['type']='Transaction'
t.add_node(transactionNode)
#For all incomming flow
for inp in data['x']['inputs']:
print inp['prev_out']['addr']
#Create a Node with properties
inNode = Node(inp['prev_out']['addr'],red=1)
inNode.property['type']='Wallet'
inNode.property['time']=data['x']["time"]
t.add_node(inNode)
#Create an edge Wallet-[weight=value of the transaction]->Transaction
edge = Edge(inNode,transactionNode,True,weight=inp['prev_out']['value'])
edge.property['type'] = inp['prev_out']['type']
t.add_edge(edge)
print "*"
#For all outgoing flow
for out in data['x']['out'] :
print out['addr']
#Create a Node with properties
outNode = Node(out['addr'],red=1)
outNode.property['type']='Wallet'
outNode.property['time']=data['x']["time"]
t.add_node(outNode)
#Create an edge Transaction-[weight=value of the transaction]->Wallet
edge = Edge(transactionNode,outNode,True,weight=out['value'])
edge.property['type'] = out['type']
t.add_edge(edge)
t.commit()
def received_message(self, m):
print "===="
inNode = []
outNode = []
data = json.loads("%s" % m)
#Get All in Nodes of the transaction
for inp in data['x']['inputs']:
print inp['prev_out']['addr']
inNode += [Node(inp['prev_out']['addr'])]
print "*"
#Get All out Nodes of the transaction
for out in data['x']['out']:
print out['addr']
outNode += [Node(out['addr'])]
#Graph All the Things !
for n in inNode:
t.add_node(n)
for o in outNode:
t.add_node(o)
t.add_edge(Edge(n, o, True))
t.commit()
def make_gephi_nodes(nbunch, rgb=(1,0,0), size=10, label_fcn = str):
'''Creates nodes specific to gephi from a list of node identifiers
Note: networkx note can be any hashable types
Visual properties available are:
(x,y,z) integer - position
(r,g,b) float[0,1] - color
color string ("0xcccccc") - color alternative to rgb
size interger - size of node on graph
'''
red, green, blue = rgb
gephi_node_dict = {x:Node(label_fcn(x), red=red, green=green, blue=blue, size=size)
for x in nbunch}
return gephi_node_dict
def received_message(self, m):
#Loading the data as json
data = json.loads("%s" % m)
print "==%s==" % data['x']['hash']
#Created the node that represent the transaction
transactionNode = Node(data['x']['hash'], blue=1)
#With some properties
for prop in [
'vin_sz', 'vout_sz', 'lock_time', 'relayed_by', 'tx_index',
'time'
]:
transactionNode.property[prop] = data['x'][prop]
#Hack to avoid "size" of the node
transactionNode.property['transaction_size'] = data['x']['size']
#we type our node
transactionNode.property['type'] = 'Transaction'
t.add_node(transactionNode)
#For all incomming flow
for inp in data['x']['inputs']:
print inp['prev_out']['addr']
#Create a Node with properties
inNode = Node(inp['prev_out']['addr'], red=1)
inNode.property['type'] = 'Wallet'
inNode.property['time'] = data['x']["time"]
t.add_node(inNode)
#Create an edge Wallet-[weight=value of the transaction]->Transaction
edge = Edge(inNode,
transactionNode,
True,
weight=inp['prev_out']['value'])
edge.property['type'] = inp['prev_out']['type']
t.add_edge(edge)
print "*"
#For all outgoing flow
for out in data['x']['out']:
print out['addr']
#Create a Node with properties
outNode = Node(out['addr'], red=1)
outNode.property['type'] = 'Wallet'
outNode.property['time'] = data['x']["time"]
t.add_node(outNode)
#Create an edge Transaction-[weight=value of the transaction]->Wallet
edge = Edge(transactionNode, outNode, True, weight=out['value'])
edge.property['type'] = out['type']
t.add_edge(edge)
t.commit()
def process_item(self, item, spider):
patent_args = {'size': 5, 'red': 1, 'green': 0, 'blue': 0}
patent_node = Node(item['publication_number'], **patent_args)
patent_node.property['type'] = 'patent'
patent_node.property['title'] = item.get('title')
patent_node.property['filing_date'] = item.get('filing_date')
patent_node.property['publication_date'] = item.get('publication_date')
patent_node.property['priority_date'] = item.get('priority_date')
patent_node.property['grant_date'] = item.get('grant_date')
patent_node.property['pdf'] = item.get('pdf')
if item['publication_number'] in self.nodes:
self.gephi.change_node(patent_node)
else:
self.gephi.add_node(patent_node)
link_args = {'size': 5, 'red': 0, 'green': 0, 'blue': 1}
for citation in item.get('citations', []):
citation_node = Node(citation, **link_args)
citation_node.property['type'] = 'link'
self.gephi.add_node(citation_node)
self.gephi.add_edge(Edge(patent_node, citation_node, True))
self.nodes.add(citation)
for cited_by in item.get('cited_by', []):
cited_by_node = Node(cited_by, **link_args)
cited_by_node.property['type'] = 'link'
self.gephi.add_node(cited_by_node)
self.gephi.add_edge(Edge(cited_by_node, patent_node, True))
self.nodes.add(cited_by)
entity_args = {'size': 5, 'red': 0, 'green': 1, 'blue': 0}
entities = set(item.get('inventors', []) + item.get('assignees', []))
for entity in entities:
entity_node = Node(entity, **entity_args)
entity_node.property['type'] = 'entity'
self.gephi.add_node(entity_node)
self.gephi.add_edge(Edge(entity_node, patent_node, True))
self.logger.info('Publishing item {}'.format(item['publication_number']))
try:
self.gephi.commit()
except ConnectionError, e:
self.logger.error(e)
def clear_visualization(self, clear_data=False):
for node_id in self.G.nodes():
self.delete_node(Node(node_id))
if clear_data:
self.G.remove_node(node_id)
self.commit()
