class GraphDB(object):
def __init__(self, database="http://localhost:7474/db/data"):
self.gdb = GraphDatabase(database)
def addPaper(self, uid, title, authors):
new_node = self.gdb.node()
new_node.labels.add('Paper')
new_node['uid'] = uid
new_node['title'] = title
new_node['authors'] = authors
def getNode(self, uid):
get_query = 'MATCH (n:Paper) WHERE n.uid=%d RETURN n'%uid
qRes = self.gdb.query(q=get_query, returns=Node)
if qRes == None:
return None
return qRes[0][0] #First element of first result is the expected node
def editPaper(self, uid, key, value):
node = self.getNode(uid)
if not node:
return False
node.set(key, value)
def deletePaper(self, uid):
delQuery = 'MATCH (n { uid: %d })-[r]-() DELETE n, r'%uid
try:
self.gdb.query(q = delQuery)
except e:
return False
return True
def setReference(self, sourceUID, targetUID):
srcNode = self.getNode(sourceUID)
targetNode = self.getNode(targetUID)
if srcNode ==None or targetNode ==None:
return False
newRel = srcNode.relationships.create("REFERENCE", targetNode)
return True
'''
Created on Apr 5, 2016
@author: Victor
'''
from neo4jrestclient.client import GraphDatabase
from neo4jrestclient.query import Q
db = GraphDatabase("http://localhost:7474/db/data/")
PERSON = 'Person'
# Crear dos nodos y una relacion del primero al segundo
firstNode = db.node( name = 'Alice', age = 25 )
secondNode = db.node( name = 'Bob', age = 30 )
knows_rel = firstNode.KNOWS(secondNode, since = 2000)
# Explorar los atributos de la relacion
print knows_rel.type, 'comienza en', knows_rel.start['name'], 'y termina en', knows_rel.end['name']
# Crear una etiqueta y asignar nodos a ella
person_label = db.labels.create(PERSON)
person_label.add(firstNode, secondNode)
# Asignar etiqueta a un nodo recien creado
thirdNode = db.node( name = 'Zach', age = 27 )
thirdNode.labels.add(PERSON)
# Filtrar nodos con una etiqueta por cierta propiedad
older_than_27 = person_label.filter(Q('age', 'gte', 27))
class TwitterGraph(object):
def _init_twitter_api(self, auth_dict, cachedir=None):
# Twitter API authentication
auth = tweepy.OAuthHandler(
auth_dict['consumer_key'],
auth_dict['consumer_secret'])
auth.set_access_token(
auth_dict['access_token'],
auth_dict['access_token_secret'])
self.TWEEPY_CACHE = tweepy.cache.FileCache(cachedir, 0)
self.api = tweepy.API(auth, cache=self.TWEEPY_CACHE)
log.info('Authenticated with Twitter')
log.info('Remaing requests this hour: %s' % self.limit)
def __init__(self, auth_dict,
dburl="http://localhost:7474/db/data/",
cachedir='.cache'):
"""Initialize Twitter API and Neo4j-link."""
self._init_twitter_api(auth_dict, cachedir)
self.gdb = GraphDatabase(dburl)
# check if indexes are ok
try:
self.gdb.nodes.indexes.get('users')
log.info('users-index OK')
except NotFoundError:
self.gdb.nodes.indexes.create('users')
log.info('users-index created')
@property
def limit(self):
return self.api.rate_limit_status()['remaining_hits']
def fetch_user_data(self, user_id):
"""Fetch user data for a given ID, return dictionary."""
log.info('Fetching user data from Twitter for ID %s' % user_id)
user = self.api.get_user(user_id)
props = user.__dict__ # user properties
del props['_api'], props['status'] # no embedded objects
props['accessed'] = datetime.datetime.now()
props['detail'] = 'full'
props['type'] = 'user'
return props
def get_user(self, user_id):
"""Get user node from graph if existing, based on ID."""
i = self.gdb.nodes.indexes.get('users')
if str(user_id).isalnum(): # numerical ID
results = i.get('user_id', user_id) # always iterable
else:
results = i.get('screen_name', user_id) # always iterable
if len(results) == 1:
log.info('Found existing users, ID %s' % user_id)
return results[0]
else:
log.info('No user in graph with ID %s' % user_id)
return None
def relationship_exists(self, start_node, end_node, reltype):
d = (start_node.id, end_node.id, reltype)
q = 'START a = node(%s), b = node(%s) MATCH a -[r:%s]-> b RETURN count(r)' % d
result = self.gdb.extensions.CypherPlugin.execute_query(q).get('data')
if result:
log.info('Found existing relationship %s -%s-> %s' % (d[0], d[2], d[1]))
return True
else:
log.info('No existing relationship %s -%s-> %s' % (d[0], d[2], d[1]))
return False
#def add_followers(self, user_node, direction='both'):
def add_subscriptions(self, user_node):
try:
user_label = user_node.get('screen_name')
except NotFoundError:
user_label = str(user_node.get('id'))
# add followers
followers = self.api.followers_ids(user_node['id'])
log.info('Found %s followers for %s' % (str(len(followers)), user_label))
for follower_id in followers:
follower_node = self.get_or_create_user(follower_id)
try:
follower_label = follower_node.get('screen_name')
except NotFoundError:
follower_label = str(follower_node.get('id'))
if not self.relationship_exists(follower_node, user_node, 'Follows'):
log.info('Adding follower %s for user %s' % (follower_label, user_label))
follower_node.relationships.create('Follows', user_node,
on=datetime.datetime.now())
# add friends
friends = self.api.friends_ids(user_node['id'])
log.info('Found %s friends for %s' % (str(len(friends)), user_node['id']))
for friend_id in friends:
friend_node = self.get_or_create_user(friend_id)
try:
friend_label = friend_node.get('screen_name')
except NotFoundError:
friend_label = str(friend_node.get('id'))
if not self.relationship_exists(user_node, friend_node, 'Follows'):
log.info('Adding friend %s for user %s' % (friend_label, user_label))
user_node.relationships.create('Follows', friend_node,
on=datetime.datetime.now())
def add_user(self, user_id):
"""Adds user to graph, based on ID or screen name."""
if not str(user_id).isalnum():
raise ValueError('Identifier must be the numerical user ID')
# skip adding user if existing & detailed
existing_user = self.get_user(user_id)
if existing_user:
if existing_user.get('detail') == 'full':
log.info('Not adding user %s, already (full) in graph' % user_id)
return existing_user
if existing_user.get('detail') == 'basic':
log.info('Not adding user %s, already (basic) in graph: updating' % user_id)
return self.update_user(existing_user)
log.info('Adding user %s to graph' % user_id)
# get and assign user data to node
props = self.fetch_user_data(user_id)
user_node = self.gdb.node(**props)
# add user node to indexes
users = self.gdb.nodes.indexes.get('users')
users['user_id'][props.get('id_str')] = user_node
users['screen_name'][props.get('screen_name')] = user_node
# add followers/following
self.add_subscriptions(user_node)
return user_node
def update_user(self, user_node):
if user_node.get('detail') == 'full':
log.info('Not updating user %s, already fully detailed' % user_node['screen_name'])
return user_node
if user_node.get('detail') == 'basic':
log.info('Updating user %s' % str(user_node['id']))
props = self.fetch_user_data(user_node['id'])
user_node.properties = props
self.add_subscriptions(user_node)
users = self.gdb.nodes.indexes.get('users')
users['screen_name'][user_node['screen_name']] = user_node
return user_node
def get_or_create_user(self, user_id):
user = self.get_user(user_id)
if not user:
log.info('User %s not found, creating user with basic info' % user_id)
user = self.gdb.node(id = user_id, detail='basic',
accessed = datetime.datetime.now(),
type='user')
users = self.gdb.nodes.indexes.get('users')
users['user_id'][user_id] = user
return user
def seed(self, users):
# TODO check for empty graph
# => stop if not empty
i = self.gdb.nodes.indexes.get('users')
for user_id in users:
log.info('Adding seed node user %s' % user_id)
user = self.add_user(user_id)
i['structure']['seeds'] = user
def degrees(self):
q = """ START user=node(*)
MATCH user <-[:Follows]-> u
RETURN user.id AS id,
user.screen_name? AS screen_name,
user.detail AS detail,
count(user) AS degree"""
results = self.gdb.extensions.CypherPlugin.execute_query(q)
df = pandas.DataFrame(results['data'], columns=results['columns'])
return df.sort('degree') # sort descending on degree
def next_user_id(self):
df = self.degrees()
selected_id = df[df['detail'] == 'basic'].tail(1)['id']
return int(selected_id)
def crawl(self):
i = self.gdb.nodes.indexes.get('users')
while self.limit > 10:
u = self.add_user(self.next_user_id())
log.info('Crawled user %s' % u.screen_name)
i['structure']['crawled'] = u
#print(class_name)
if class_name not in ['crime', 'объект: противозаконная деятельность'.decode('utf-8'), 'события: противозаконная деятельность'.decode('utf-8'),
'объект: конфликты'.decode('utf-8'), 'личное взаимодействие: физическое насилие'.decode('utf-8')]:
frames.remove(frame)
#try to insert frame in Neo4J
else:
o1, o2, ev = None, None, None
for slot in frame.findall('slot'):
if slot.attrib['name'] in ['person_object', 'Object1', 'Obj1', 'object1']: #crime
o1 = slot.text
if slot.attrib['name'] in ['person_subject', 'Object2']: #victim
o2 = slot.text
if slot.attrib['name'] in ['Event', 'event']:
ev = slot.text
# print(slot.attrib['name'])
# print(slot.text)
if o1 is not None and o2 is not None:
obj1 = gdb.node(name=o1)
obj2 = gdb.node(name=o2)
# obj1.labels.add(["Person", "Crime"])
# obj2.labels.add(["Person", "Victim"])
obj1.labels.add(["Crime"])
obj2.labels.add(["Victim"])
if ev is not None:
obj1.Crime(obj2, event=ev)
else:
obj1.Crime(obj2)
#now let's insert all features of document
'''
Created on Nov 5, 2016
@author: Victor
'''
from neo4jrestclient import traversals
from neo4jrestclient.client import GraphDatabase
from neo4jrestclient.query import Q
db = GraphDatabase('http://localhost:7474/db/data/')
neo = db.node(name='Thomas Anderson', age=29)
neo.labels.add('Character')
trinity = db.node(name='Trinity')
trinity.labels.add('Character')
morpheus = db.node(name='Morpheus', rank='Captain')
morpheus.labels.add('Character')
cypher = db.node(name='Cypher')
cypher.labels.add('Character')
agent_smith = db.node(name='Agent Smith', language='C++', version='1.0b')
agent_smith.labels.add('Character')
architect = db.node(name='The Architect')
architect.labels.add('Character')
neo.KNOWS(trinity, age='3 days')
morpheus.KNOWS(trinity, age='12 years')
neo.KNOWS(morpheus)
morpheus.KNOWS(cypher, disclosure='public')
cypher.KNOWS(agent_smith, disclosure='secret', age='6 months')
agent_smith.CODED_BY(architect)
'''
Created on Apr 5, 2016
@author: Victor
'''
from neo4jrestclient.client import GraphDatabase
db = GraphDatabase('http://localhost:7474/db/data/')
index1 = db.nodes.indexes.create('index1')
firstNode = db.node(name='Madrid', country='Spain')
secondNode = db.node(name='Zaragoza', country='Spain')
index1['places']['madrid'] = firstNode
index1['places']['zaragoza'] = secondNode
for node in index1.query('places', '?a*'):
print node['name']
index1.delete('places', None, firstNode)
index1.delete('places', 'zaragoza', secondNode)
for node in index1.query('places', '*'):
print node['name']
index1.delete()
(root_node, tline) = tline[hour]
else:
tline[hour] = (gdb.node(type="TIMELINE"), {})
root_node.relationships.create("NEXT_LEVEL", tline[hour][0], hour=hour)
(root_node, tline) = tline[hour]
root_node.relationships.create("TIMELINE_INSTANCE", node, timestamp=timestamp)
# -----------------------------------------------------
gdb = GraphDatabase(NEO4J_REST)
tagsidx = gdb.nodes.indexes.create(name="tags_%s" % RUN_NAME, type="fulltext")
REF_NODE = gdb.node[0]
RUN = gdb.node(name=RUN_NAME, type='RUN')
REF_NODE.relationships.create("HAS_RUN", RUN)
TLINE = gdb.node(name='TIMELINE', type='TIMELINE', start=START_TIME, stop=STOP_TIME)
RUN.relationships.create("HAS_TIMELINE", TLINE)
TAG_DICT = {}
EDGE_DICT = {}
frame_count = 0
prev_frame = None
tags = set()
edges = set()
frame_tags = []
frame_edges = []
'''
Created on Apr 5, 2016
@author: Victor
'''
from neo4jrestclient import traversals
from neo4jrestclient.client import GraphDatabase
from neo4jrestclient.query import Q
db = GraphDatabase('http://localhost:7474/db/data/')
home = db.node(name='Home')
neo = db.node(name='Thomas Anderson', age=29)
trinity = db.node(name='Trinity')
morpheus = db.node(name='Morpheus', rank='Captain')
cypher = db.node(name='Cypher')
agent_smith = db.node(name='Agent Smith', language='C++', version='1.0b')
architect = db.node(name='The Architect')
home.NEO_NODE(neo)
neo.KNOWS(trinity, age='3 days')
morpheus.KNOWS(trinity, age='12 years')
neo.KNOWS(morpheus)
morpheus.KNOWS(cypher, disclosure='public')
cypher.KNOWS(agent_smith, disclosure='secret', age='6 months')
agent_smith.CODED_BY(architect)
traversal_description = traversals.TraversalDescription()\
.relationships('KNOWS', traversals.RelationshipDirection.OUTGOING)\
'''
Created on Apr 5, 2016
@author: Victor
'''
from neo4jrestclient.client import GraphDatabase
from neo4jrestclient.query import Q
from neo4jrestclient.constants import DESC
db = GraphDatabase("http://localhost:7474/db/data/")
nodes = []
nodes.append(db.node(name = 'Robert Parsons', code=1))
nodes.append(db.node(name = 'William Gates', code=5))
nodes.append(db.node(name = 'Sergey Brin', code=2))
nodes.append(db.node(name = 'Gordon Moore', code=4))
nodes.append(db.node(name = 'Steven Paul Jobs', code=3))
lookup = Q('name', istartswith='s')
s_founders = db.nodes.filter(lookup)
print 'Comenzando con S:'
for node in s_founders:
print '-', node['name']
lookup = Q('name', istartswith='s') & Q('name', iendswith='n')
sn_founders = db.nodes.filter(lookup)
print
print 'Comenzando con S y terminando con N:'
"ROAD_TYPE": ["Other", "Motorway"], "LIGHT_CONDITIONS": ["Day", "Dusk"], "WEATHER": ["Rain"]},
{"RECFILENAME": "20141215_1540_{08627FB2-FDC3-414E-92B3-EBE39884DA8F}.rrec", "PROJECT": "MFC500", "FUNCTION": "SR", "DEPARTMENT": "DEV", "COUNTRY": ["TAIWAN", "Austria"],
"ROAD_TYPE": ["Country_Road", "Highway"], "LIGHT_CONDITIONS": ["Dusk", "Night"], "WEATHER": ["Dry"]},
]
# data_lst_dct = ''
with open("ldss.json") as outfile:
# self.config_schema = OrderedDict(json.load(outfile))
data_lst_dct = json.load(outfile)
for ech_rcord_dict in data_lst_dct:
print ("ech_rcord_dict :: ", ech_rcord_dict["RECFILENAME"])
# Create recording node
rec_node = gdb.node(name=ech_rcord_dict["RECFILENAME"])
rec_node.labels.add("RecordingName")
print ("Created rec node.....")
# Create common data node
cmn_data_node = gdb.node(project=ech_rcord_dict["PROJECT"], function=ech_rcord_dict["FUNCTION"],
department=ech_rcord_dict["DEPARTMENT"])
cmn_data_node.labels.add("CommonData")
# Establish relationship with rec and common data node
rec_node.relationships.create("cd", cmn_data_node)
print ("Created common data node.....")
# Create country node
cntry_node = gdb.node(name=ech_rcord_dict["COUNTRY"])
cntry_node.labels.add("Country")
# Establish relationship with rec and country node
rec_node.relationships.create("c", cntry_node)
node['year'] = year
node['month'] = month
node['day'] = day
node['hour'] = hour
node['minute'] = minute
node['second'] = second
# -----------------------------------------------------
gdb = GraphDatabase(NEO4J_REST)
actorsidx = gdb.nodes.indexes.create(name="actors_%s" % RUN_NAME, type="fulltext")
REF_NODE = gdb.node[0]
RUN = gdb.node(name=RUN_NAME, type='RUN')
REF_NODE.relationships.create("HAS_RUN", RUN)
TLINE = gdb.node(name='TIMELINE', type='TIMELINE', start=START_TIME, stop=STOP_TIME)
RUN.relationships.create("HAS_TIMELINE", TLINE)
ACTOR_DICT = {}
INTERACTION_DICT = {}
frame_count = 0
prev_frame = None
actors = set()
interactions = set()
frame_actors = []
frame_interactions = []
