def update_info_and_links():
print 'updating show info'
authenticate("localhost:7474", "neo4j", "1234")
graph = Graph(GRAPH_CONNECTION_STRNIG)
results = graph.cypher.stream("match (s:Show) return id(s) as eid,s.id")
start_id = 0
for record in results:
if int(record['s.id']) < start_id:
continue
node_show = graph.node(record['eid'])
result_dict = {}
success = True
while success:
try:
show_info_e_list = requests.get(
'http://services.tvrage.com/feeds/full_show_info.php?sid={0}'.format(node_show['id']))
result_dict = xmltodict.parse(show_info_e_list.text)
omdb_show_info = requests.get(
'http://www.omdbapi.com/?t={0}&y=&plot=full&r=json'.format(node_show['name']))
dict_omdb_show_info = json.loads(omdb_show_info.text)
if dict_omdb_show_info['Response'] == 'True':
for key, value in dict_omdb_show_info.iteritems():
node_show[key] = value
success = False
except ValueError as e:
logger.exception("Value error")
continue
except Exception as e:
logger.exception("Some network issue: will try again")
success = True
print str(node_show['name'])
# info
node_show['started'] = result_dict['Show'].get('started', None)
node_show['ended'] = result_dict['Show'].get('ended', None)
node_show['image'] = result_dict['Show'].get('image', None)
node_show['status'] = result_dict['Show'].get('status', None)
node_show.push()
#Country
from_country = result_dict['Show'].get('origin_country', 'unknown')
node_country = graph.merge_one("Country", 'country', from_country)
node_country.push()
show_from_country = Relationship(node_show, "from", node_country)
graph.create(show_from_country)
#Genres
if result_dict['Show'].get('genres', None) is not None:
genre_list = []
if type(result_dict['Show']['genres']['genre']) is list:
genre_list = result_dict['Show']['genres']['genre']
else:
genre_list.append(result_dict['Show']['genres']['genre'])
for genre in genre_list:
node_genre = graph.merge_one("Genre", 'name', genre)
node_genre.push()
show_of_genre = Relationship(node_show, "of genre", node_genre)
graph.create(show_of_genre)
"""
try:
print node_show['started']
a = node_show['started'].split("/")
if int(a[len(a)-1]) < 2000:
continue
except Exception:
continue
"""
#Seasons
season_list = []
if result_dict['Show'].get('Episodelist', None) is None:
continue
if type(result_dict['Show']['Episodelist']['Season']) is list:
season_list = result_dict['Show']['Episodelist']['Season']
else:
season_list.append(result_dict['Show']['Episodelist']['Season'])
for season in season_list:
node_season = Node.cast('Season', {'no': season['@no']})
graph.create(node_season)
show_season = Relationship(node_show, "has", node_season)
graph.create(show_season)
#Episodes
episode_list = []
if type(season['episode']) is list:
episode_list = season['episode']
else:
episode_list.append(season['episode'])
count = 1
for episode in episode_list:
node_episode = Node.cast('Episode', {
'airdate': episode.get('airdate', None),
'epnum': count,
'screencap': episode.get('screencap', None),
'title': episode.get('title', None)
})
graph.create(node_episode)
success = True
while success:
try:
omdb_episode_info = requests.get('http://www.omdbapi.com/?t={0}&Season={1}&Episode={2}'
.format(node_show['name'],
node_season['no'],
node_episode['epnum']))
dict_omdb_episode_info = json.loads(omdb_episode_info.text)
if dict_omdb_episode_info['Response'] == 'True':
for key, value in dict_omdb_episode_info.iteritems():
node_episode[key] = value
node_episode.push()
success = False
except ValueError as e:
logger.exception("Value error")
continue
except Exception as e:
logger.exception("Some network issue: will try again")
success = True
try:
search = node_show['name'] + ' s' + str(node_season['no']).zfill(2) + 'e' + str(node_episode['epnum']).zfill(2)
print search
#links
search_numbers = [3552639851, 8556419051, 2649486255, 7079685853, 8416818254, 1870757059,
1731156253, 4545021852, 6021755051, 8975221455]
for n in search_numbers:
links_from_google = requests.get(
'https://www.googleapis.com/customsearch/v1element?key=AIzaSyCVAXiUzRYsML1Pv6RwSG1gunmMikTzQqY&rsz=small&num=10&hl=en&prettyPrint=false&source=gcsc&gss=.com&sig=cb6ef4de1f03dde8c26c6d526f8a1f35&cx=partner-pub-2526982841387487:{1}'
'&q={0}&googlehost=www.google.com&oq={0}'.format(search, n))
dict_from_google = json.loads(links_from_google.text)
for result in dict_from_google['results']:
node_link = Node.cast('Link', {
'host': result.get('visibleUrl', None),
'url': result['url']
})
graph.create(node_link)
link_episode = Relationship(node_episode, "has", node_link)
graph.create(link_episode)
except Exception, err:
logger.exception("error grom google part")
show_episode = Relationship(show_season, "has", node_episode)
graph.create(show_episode)
count = count + 1
if a and b:
ENDPOINT = "http://*****:*****@localhost:7474/db/data/"
request = {
"to":ENDPOINT+"node/"+str(b._id),
"max_depth": int(sys.argv[3]),
"relationships": {
"type":"LINKS",
"direction":"out"
},
"algorithm":"allSimplePaths"
}
r = requests.post(ENDPOINT+"node/"+str(a._id)+"/paths", data=json.dumps(request))
# print r.json()
if r.status_code == 200:
for path in r.json():
print "Path:"
for node in path['nodes']:
# print node
retrieved_node = graph.node(node.split('/node/')[1])
print retrieved_node.properties['title'], "->"
print "--------------------"
else:
print "Something went wrong."
print sys.exc_info()[0]
else:
if not a:
print sys.argv[1], "not found!"
else:
print sys.argv[2], "not found!"
def update_show_info_old():
print 'updating show info'
authenticate("localhost:7474", "neo4j", "1234")
graph = Graph(GRAPH_CONNECTION_STRNIG)
results = graph.cypher.stream("match (s:Show) return id(s) as eid,s.id")
start_id = 764
for record in results:
if int(record['s.id']) < start_id:
continue
node_show = graph.node(record['eid'])
result_dict = {}
success = True
while success:
try:
show_info_e_list = requests.get(
'http://services.tvrage.com/feeds/full_show_info.php?sid={0}'.format(node_show['id']))
result_dict = xmltodict.parse(show_info_e_list.text)
omdb_show_info = requests.get(
'http://www.omdbapi.com/?t={0}&y=&plot=full&r=json'.format(node_show['name']))
dict_omdb_show_info = json.loads(omdb_show_info.text)
if dict_omdb_show_info['Response'] == 'True':
for key, value in dict_omdb_show_info.iteritems():
node_show[key] = value
success = False
except ValueError as e:
logger.exception("Value Error")
continue
except Exception as e:
logger.exception("Some network issue, will try again")
success = True
print str(node_show['name'])
# info
node_show['started'] = result_dict['Show'].get('started', None)
node_show['ended'] = result_dict['Show'].get('ended', None)
node_show['image'] = result_dict['Show'].get('image', None)
node_show['status'] = result_dict['Show'].get('status', None)
node_show.push()
#Country
from_country = result_dict['Show'].get('origin_country', 'unknown')
node_country = graph.merge_one("Country", 'country', from_country)
node_country.push()
show_from_country = Relationship(node_show, "from", node_country)
graph.create(show_from_country)
#Genres
if result_dict['Show'].get('genres', None) is not None:
genre_list = []
if type(result_dict['Show']['genres']['genre']) is list:
genre_list = result_dict['Show']['genres']['genre']
else:
genre_list.append(result_dict['Show']['genres']['genre'])
for genre in genre_list:
node_genre = graph.merge_one("Genre", 'name', genre)
node_genre.push()
show_of_genre = Relationship(node_show, "of genre", node_genre)
graph.create(show_of_genre)
"""try:
print node_show['started']
a = node_show['started'].split("/")
if int(a[len(a)-1]) < 2000:
continue
except Exception:
continue
"""
#Seasons
season_list = []
if result_dict['Show'].get('Episodelist', None) is None:
continue
if type(result_dict['Show']['Episodelist']['Season']) is list:
season_list = result_dict['Show']['Episodelist']['Season']
else:
season_list.append(result_dict['Show']['Episodelist']['Season'])
for season in season_list:
node_season = Node.cast('Season', {'no': season['@no']})
graph.create(node_season)
show_season = Relationship(node_show, "has", node_season)
graph.create(show_season)
#Episodes
episode_list = []
if type(season['episode']) is list:
episode_list = season['episode']
else:
episode_list.append(season['episode'])
count = 1
for episode in episode_list:
node_episode = Node.cast('Episode', {
'airdate': episode.get('airdate', None),
'epnum': count,
'screencap': episode.get('screencap', None),
'title': episode.get('title', None)
})
graph.create(node_episode)
success = True
while success:
try:
omdb_episode_info = requests.get('http://www.omdbapi.com/?t={0}&Season={1}&Episode={2}'
.format(node_show['name'],
node_season['no'],
node_episode['epnum']))
dict_omdb_episode_info = json.loads(omdb_episode_info.text)
if dict_omdb_episode_info['Response'] == 'True':
for key, value in dict_omdb_episode_info.iteritems():
node_episode[key] = value
node_episode.push()
success = False
except ValueError as e:
logger.exception("Value error")
continue
except Exception as e:
logger.exception("network issue: wil try again")
success = True
show_episode = Relationship(show_season, "has", node_episode)
graph.create(show_episode)
count = count + 1
print 'end updating show info'
r1 = floor(tot_len * (i + 1) * 0.01)
if l1-1 == r1:
l1 = r1
cursor1 = db.get_cursor()
cursor1.execute(query1,(str(tmony),str(l1),str(r1)))
if(i == 50):
while True:
row = cursor1.fetchone()
if not row:
break
# Process each of these UNIQUE parent id's here
if row:
term_node = graph.find_one("TERM", "term_id", row["parent_id"])
segment_node_id = graph.cypher.execute(query2neo4id, {"tmony": tmony, "segment": str(i+1)})
segment_node = graph.node(segment_node_id[0][0])
has_term_relationship = Relationship(segment_node, "HAS_TERM", term_node)
is_in_relationship = Relationship(term_node, "IS_IN", segment_node)
graph.create(has_term_relationship)
graph.create(is_in_relationship)
cursor1.close()
count += 1
__author__ = 'SU915198'
from py2neo import Node, Rel, Graph
# node_name = Node("http://localhost:7474/db/data")
graph = Graph("http://localhost:7474/")
n = graph.node("People")
class GraphDB:
def __init__(self,username, password, server, port):
'''
username = '******'
password = '******'
server = 'localhost'
port = '7474'
'''
self.username = username
self.password = password
self.server = server
self.port = port
self.con_url = 'http://'+username+':'+password+'@'+server+':'+port+'/db/data/'
self.graph = Graph(self.con_url)
##TODO: move these to constants.py when IPYTHON not required
self.metaprops = {
'RESOLVEDUUID':'_resolvedWithUUID_',
'RESOLVEDRELID':'_resolvedWithRELID_',
'RESOLVEDHENID':'_resolvedWithHENID_', ##for hyper edge node
'RESOLVEDHERID':'_resolvedWithHERID_', ##for hyper edge relation
}
#Done: Multiple each time want to ask something
#Else: works on old data only
##Wont use this
def getGraph(self):
return self.graph
##NOT TO BE USED
#this is how we use the intenal ids of the graph
##should we use it?? most people say no
## anyways, made the method for future use
## Use : getNodeByInternalId(graph, 152)
def getNodeByInternalId(self,id):
a = self.graph.node(id) #getting by id given internally by neo4j
a.pull()
return a
##NOT TO BE USED
## similar to above
## Use : getRelByInternalId(graph, 4)
def getRelByInternalId(self,id):
a = self.graph.relationship(id)
a.pull()
return a
def getNodeByUniqueID(self, uniquelabel, idName, idVal, isIDString=False):
##TODO: move uuid to props
query = "match (n:"+uniquelabel+" {"
if isIDString:
query = query+ idName+":'"+str(idVal)+"'}) return n"
else:
query = query+ idName+":"+str(idVal)+"}) return n"
rc = self.graph.cypher.execute(query)
return rc[0][0]
def getRelationByUniqueID(self, idName, idVal, isIDString=False):
##TODO: move uuid to props
query = "match ()-[r {"
if isIDString:
query = query+ idName+":'"+str(idVal)+"'}]->() return r"
else:
query = query+ idName+":"+str(idVal)+"}]->() return r"
rc = self.graph.cypher.execute(query)
return rc[0][0]
def isPropList(self, node, prop):
return type(node[prop]) is list
##has a counter brother in nexusapis flask app
def isValidNonMetaProp(self, propname):
if propname[0]=='_' or propname[-1]=='_':
return False
return True
##copy meta = True
def copyNode(self, node, copymeta = True, exceptions = []):
#exceptions are the props that should be included no matter what, if they have underscore or not!
naya = Node()
for label in node.labels:
naya.labels.add(label)
for prop in exceptions:
if prop in node.properties:
naya[prop] = node[prop]
for prop in node.properties:
if not copymeta:
if self.isValidNonMetaProp(prop):
naya[prop] = node[prop]
else:
naya[prop] = node[prop]
return naya
def copyNodeAsItIs(self, node):
return self.copyNode(node)
# naya = Node()
# for label in node.labels:
# naya.labels.add(label)
# for prop in node.properties:
# naya[prop] = node[prop]
# return naya
def copyNodeWithoutMeta(self, node, exceptions=[]):
return self.copyNode(node, copymeta = False, exceptions = exceptions)
def copyRelation(self, rel, copymeta = True, rel_exceptions = [], node_exceptions = []):
start_node = ''
end_node = ''
if not copymeta:
start_node = self.copyNodeWithoutMeta(rel.start_node, exceptions = node_exceptions)
end_node = self.copyNodeWithoutMeta(rel.end_node, exceptions = node_exceptions)
else:
start_node = self.copyNodeAsItIs(rel.start_node)
end_node = self.copyNodeAsItIs(rel.end_node)
reltype = rel.type
nayarel = Relationship(start_node, reltype, end_node)
for prop in rel_exceptions:
if prop in rel.properties:
nayarel[prop] = rel[prop]
for prop in rel.properties:
if not copymeta:
if self.isValidNonMetaProp(prop):
nayarel[prop] = rel[prop]
else:
nayarel[prop] = rel[prop]
return nayarel
def copyRelationAsItIs(self, rel):
return self.copyRelation(rel)
# start_node = g.copyNodeAsItIs(rel.start_node)
# end_node = g.copyNodeAsItIs(rel.end_node)
# reltype = rel.type
# naya = Relationship(start_node, reltype, end_node)
# for prop in rel.properties:
# naya[prop] = rel[prop]
# return naya
def copyObjectAsItIs(self, kind, obj):
if kind == 'relation':
return self.copyRelationAsItIs(obj)
if kind == 'node':
return self.copyNodeAsItIs(obj)
return None
def copyRelationWithoutMeta(self, rel, rel_exceptions = [], node_exceptions = []):
return self.copyRelation(rel, copymeta = False, rel_exceptions = rel_exceptions, node_exceptions = node_exceptions)
def getDirectlyConnectedEntities(self, idname, idval, uniquelabel='', isIDString = True):
'''
given idname like uuid, henid and uniquelabel like entity or hyperedgenode
gives us the connected nodes of this node
returns nodes which have uuids only
note: works for hyperedgenodes only for now
#TODO: extend so that this info can be shown on view page
'''
from app.constants import LABEL_ENTITY
invertedComma = ''
if isIDString:
invertedComma = "'"
query = "match (n:%s {%s:%s%s%s})-[]-(p:%s) return distinct(p)"
query = query %(uniquelabel, idname, invertedComma, idval, invertedComma, LABEL_ENTITY)
results = self.graph.cypher.execute(query)
enlist = []
for res in results:
enlist.append(res[0])
return enlist
def getDirectlyConnectedRelations(self, idname, idval, uniquelabel='', isIDString = True,outgoing=True):
'''
given idname like uuid, henid and uniquelabel like entity or hyperedgenode
gives us the connected nodes of this node
returns nodes which have uuids only
note: works for hyperedgenodes only for now
#TODO: extend so that this info can be shown on view page
'''
from app.constants import LABEL_ENTITY
invertedComma = ''
if isIDString:
invertedComma = "'"
query = ''
if outgoing:
query = "match (n:%s {%s:%s%s%s})-[p]->(:%s) return distinct(p)"
else:
query = "match (n:%s {%s:%s%s%s})<-[p]-(:%s) return distinct(p)"
query = query %(uniquelabel, idname, invertedComma, idval, invertedComma, LABEL_ENTITY)
results = self.graph.cypher.execute(query)
rellist = []
for res in results:
rellist.append(res[0])
return rellist
##example of some text input: (n154346:businessperson:person:politician {name:"Anita",uuid:1234})
##Usage: deserializeNode('''(n154346:businessperson:person:politician {name:"Anita",uuid:1234})''')
def deserializeNode(self, nodeText):
pos = nodeText.find(' ')
#get the labels in a set
startText = nodeText[1:pos]
allLabels = startText.split(':')[1:]
allLabels = set(allLabels) #set is imp
#get the props in a dict
endText = nodeText[pos+1:-1]
endTextWB = endText[1:-1]
#print endText
#print endTextWB
propList = endTextWB.split(",")
propsDict = {}
for x in propList:
propval = x.split(":")
#for handling the single inverted comma problem
prop = propval[0]
val = propval[1]
if val[0]=="'" and val[-1]=="'":
val=val[1:-1]
#for handling the double inverted comma problem
if val[0]=='"' and val[-1]=='"':
val=val[1:-1]
propsDict[prop]=val
#print propsDict
#creating the node from parsedText
node = Node()
for x in allLabels:
node.labels.add(x)
for x in propsDict:
node[x] = propsDict[x]
print node
return node
##MAJOR TODO:
##work to do add a form where they can create a node
#get a node's page by uuid
#also get node's relations in form of graph, embed that graph
#edit a node
#edit a relation
#dlete a node
#delete a relation
#moderate any change --> how to do that --> where will this lie!
#Note the diff between now and then
def generateSearchData(self, idname, idval, isIDString, getList=False):
from app.utils.commonutils import Utils
comp = self.getNodeByUniqueID('entity', idname, idval, isIDString)
##comp is the node in question
keywords = ''
labels = ''
aliases_to_return = ''
if getList:
keywords = []
labels = []
aliases_to_return = []
quotes = "" ##the ' wre reudundant from beginning
for prop in comp.properties:
##begins with underscore ignore
if prop!='uuid' and prop!='aliases' and prop[0]!='_':
currvalue = str(comp.properties[prop])
currvalue = Utils.processString(currvalue)
if prop!='uuid' and len(currvalue)>3:
if not getList:
keywords = keywords + quotes +currvalue + quotes+","
else:
keywords.append(currvalue)
neighbours = self.getDirectlyConnectedEntities(idname, idval,
'entity', isIDString)
for rel in neighbours:
currvalue = str(rel.properties['name'])
currvalue = Utils.processString(currvalue)
if not getList:
keywords = keywords + quotes +currvalue + quotes+","
else:
keywords.append(rel.properties['name'])
# keywords = keywords + '"'
# labels = ''
for label in list(comp.labels):
if not getList:
labels = labels + quotes + label + quotes+","
else:
labels.append(label)
# labels=labels + '"'
aliases = []
if aliases is None or aliases == []:
aliases = [comp['name']]
##patch
from app.utils.commonutils import Utils
aliases = Utils.merge(comp['aliases'],comp['name'])
print 'allllllllllllllaaaaaaaa'
print aliases
# if type(aliases) is not list: ##fix for just one aliases, though wont be req
# aliases = [aliases]
# ##patch
for alias in aliases:
if not getList:
aliases_to_return = aliases_to_return + quotes +str(Utils.processString(alias)) + quotes+","
else:
aliases_to_return.append(alias)
# aliases =aliases_to_return + '"'
# name = '"'+comp.properties['name']+'"'
name = comp.properties['name']
print 'aliases_to_return, aliases, aliases_to_return'
print aliases_to_return
print keywords
if len(keywords)>50:
keywords=keywords[:50]
return name, labels, aliases_to_return, keywords
class DatabaseConnection(object):
def __init__(self):
authenticate("localhost:7474", "neo4j", "admin")
self.connection = Graph()
def add_person(self, person_name, year_of_birth):
self.connection.create(Node("Person", name=person_name, born=year_of_birth))
def add_relationship(self, start_id, end_id, relationship_type):
start_node = self.connection.node(start_id)
end_node = self.connection.node(end_id)
self.connection.create(Relationship(start_node, relationship_type, end_node))
def remove_relationship(self, relationship_id):
self.connection.cypher.stream(
"MATCH (n) - [r] - () WHERE ID(r) = {} DELETE r".format(relationship_id))
def remove_person(self, person_id):
self.connection.cypher.stream(
"MATCH(p:Person) where ID(p) = {} OPTIONAL MATCH(p) - [r] - () DELETE r, p".format(
person_id))
def add_person_property(self):
pass
def update_person(self, person_id, person_properties):
node_to_update = self.get_neo4j_person(person_id)
keys = node_to_update.properties.keys()
for key in keys:
del node_to_update.properties[key]
if person_properties:
for key in person_properties:
node_to_update.properties[key] = person_properties[key]
node_to_update.push()
def update_relationship(self, relationship_id, relationship_properties):
relationship_to_update = self.get_neo4j_relationship(relationship_id)
keys = relationship_to_update.properties.keys()
for key in keys:
del relationship_to_update.properties[key]
if relationship_properties:
for key in relationship_properties:
relationship_to_update.properties[key] = relationship_properties[key]
relationship_to_update.push()
def get_neo4j_person(self, person_id):
single_node_list = self.connection.cypher.stream(
"MATCH(p:Person) where ID(p) = {} RETURN p".format(person_id))
for a_node in single_node_list:
return a_node[0]
def get_person(self, person_id):
neo_node = self.get_neo4j_person(person_id)
return person.Person(str(neo_node.uri).rsplit('/', 1)[-1],
neo_node.properties["name"],
neo_node.properties["born"],
neo_node.properties)
def get_all_persons(self):
nodes = list()
for n in self.connection.cypher.stream("START z=node(*) RETURN z"):
new_node = person.Person(str(n[0].uri).rsplit('/', 1)[-1], n[0].properties["name"],
n[0].properties["born"], n[0].properties)
nodes.append(new_node)
return nodes
def get_relationship(self, relationship_id):
neo_relationship = self.get_neo4j_relationship(relationship_id)
relationship_to_return = relationship.Relationship(
relationship_id=str(neo_relationship.uri).rsplit('/', 1)[-1],
start_node=neo_relationship.start_node,
end_node=neo_relationship.end_node,
relationship_type=neo_relationship.type,
properties=neo_relationship.properties)
return relationship_to_return
def get_neo4j_relationship(self, relationship_id):
single_relationship_list = self.connection.cypher.stream(
"MATCH (n) - [r] - () WHERE ID(r) = {} RETURN r"
.format(relationship_id))
for a_relation in single_relationship_list:
return a_relation[0]
def get_all_relationships(self):
relations = list()
for relation in self.connection.cypher.stream("Match (a)-[r]->(b) return r"):
new_relationship = relationship.Relationship(
relationship_id=str(relation[0].uri).rsplit('/', 1)[-1],
start_node=relation[0].start_node,
end_node=relation[0].end_node,
relationship_type=relation[0].type,
properties=relation[0].properties)
relations.append(new_relationship)
return relations
def get_child_generations_for_person(self, person_id):
return_value = next(self.connection.cypher.
stream(
"MATCH p=(r:Person)<-[:CHILD_OF*1..20]-(x) WHERE ID(r) = {} RETURN max(length(p))".
format(person_id)))[0]
return_value = return_value if return_value else 0
return return_value
def get_parent_generations_for_person(self, person_id):
return_value = next(self.connection.cypher.
stream(
"MATCH p=(r:Person)-[:CHILD_OF*1..20]->(x) WHERE ID(r) = {} RETURN max(length(p))".
format(person_id)))[0]
return_value = return_value if return_value else 0
return return_value
def is_descendant_of(self, descendant_person_id, ancestor_person_id):
path_length = next(self.connection.cypher.
stream(
"MATCH p=(r:Person)-[:CHILD_OF*1..20]->(q:Person) WHERE ID(r) = {} and ID(q) = {}"
"RETURN max(length(p))".
format(descendant_person_id, ancestor_person_id)))[0]
return True if path_length else False
def is_married(self, person_id):
path_length = next(self.connection.cypher.
stream(
"MATCH p=(r:Person)-[:MARRIED]-(x) WHERE ID(r) = {} RETURN max(length(p))".
format(person_id)))[0]
return True if path_length else False
def get_spouses_of_person(self, person_id):
spouse_list = list()
for spouse in self.connection.cypher.stream(
"MATCH p=(r:Person)-[:MARRIED]-(q:Person) WHERE ID(r) = {} "
"RETURN q".format(person_id)):
spouse_list.append(str(spouse[0].uri).rsplit('/', 1)[-1])
return spouse_list
def get_parents_of_person(self, person_id):
parents_list = list()
for parent in self.connection.cypher.stream(
"MATCH p=(r:Person)-[:CHILD_OF]->(q:Person) WHERE ID(r) = {} RETURN q".
format(person_id)):
parents_list.append(str(parent[0].uri).rsplit('/', 1)[-1])
return parents_list if parents_list else False
def get_children_of_person(self, person_id):
children_list = list()
for parent in self.connection.cypher.stream(
"MATCH p=(r:Person)<-[:CHILD_OF]-(q:Person) WHERE ID(r) = {} RETURN q".
format(person_id)):
children_list.append(str(parent[0].uri).rsplit('/', 1)[-1])
return children_list if children_list else False
def get_relationships_of_a_node(self, person_id):
relationship_list = list()
return_value = next(self.connection.cypher.stream(
"MATCH p=(person:Person)-[r]-() WHERE ID(person) = {} WITH collect(r) AS rs RETURN rs".
format(person_id)))
start_place = 0
while True:
start_place = str(return_value).find("relationship/", start_place) + len(
"relationship/")
if start_place == -1 + len("relationship/"):
break
end_place = str(return_value).find("'", start_place)
relationship_list.append(
self.get_relationship(str(return_value)[start_place:end_place]))
start_place = end_place
return relationship_list
class Neo4jModel:
def __init__(self):
self.graph = Graph()
def create(self):
self.graph.schema.create_uniqueness_constraint("Region", "name")
self.graph.schema.create_uniqueness_constraint("Court", "name")
self.graph.schema.create_uniqueness_constraint("Court_Decision_Type", "name")
self.graph.schema.create_uniqueness_constraint("Court_Judgement_Type", "name")
self.graph.schema.create_uniqueness_constraint("Case", "id")
self.graph.schema.create_uniqueness_constraint("Chairman", "name")
def region(self, region_name):
__region = self.graph.merge_one("Region", "name", region_name)
__region.push()
return __region
def court(self, court_name, region_name):
__court = self.graph.merge_one("Court", "name", court_name)
__court.push()
self.graph.create_unique(Relationship(__court, "SITUATED_IN", self.region(region_name)))
return __court
def chairman(self, chairman_name):
__chairman = self.graph.merge_one("Chairman", "name", chairman_name)
__chairman.push()
return __chairman
def decision_type(self, decision_type_name):
__decision_type = self.graph.merge_one("Court_Decision_Type", "name", decision_type_name)
__decision_type.push()
return __decision_type
def judgement_type(self, judgement_type_name):
__judgement_type = self.graph.merge_one("Court_Judgement_Type", "name", judgement_type_name)
__judgement_type.push()
return __judgement_type
def case(self, court_case, region_name):
__case = self.graph.merge_one("Case", "id", court_case.decision_number)
__case["reg_date"] = __timestamp__(court_case.reg_date)
__case["law_date"] = __timestamp__(court_case.law_date)
__case["link"] = court_case.link
__case["text"] = court_case.text
__case["case_number"] = court_case.case_number
self.graph.create_unique(Relationship(__case, "RULED_BY", self.court(court_case.court_name, region_name)))
self.graph.create_unique(Relationship(__case, "CARRIED_BY", self.chairman(court_case.chairman)))
self.graph.create_unique(Relationship(__case, "OF_JUDGEMENT_TYPE", self.judgement_type(court_case.vr_type)))
self.graph.create_unique(Relationship(__case, "OF_DECISION_TYPE", self.decision_type(court_case.cs_type)))
__case.push()
return __case
def change_date(self):
query = "MATCH (n:Case) WHERE NOT (n.law_date='') RETURN n LIMIT 5"
id_list = []
for n in self.graph.cypher.execute(query):
id_list.append(n[0].__str__()[2:].split(':')[0]) # getting an id
for _id in id_list:
n = self.graph.node(str(_id))
n['law_date'] = __timestamp__(n['law_date'])
n.push()
print(n)
class Neo4jModel:
def __init__(self):
self.graph = Graph()
def create(self):
self.graph.schema.create_uniqueness_constraint("Region", "name")
self.graph.schema.create_uniqueness_constraint("Court", "name")
self.graph.schema.create_uniqueness_constraint("Court_Decision_Type",
"name")
self.graph.schema.create_uniqueness_constraint("Court_Judgement_Type",
"name")
self.graph.schema.create_uniqueness_constraint("Case", "id")
self.graph.schema.create_uniqueness_constraint("Chairman", "name")
def region(self, region_name):
__region = self.graph.merge_one("Region", "name", region_name)
__region.push()
return __region
def court(self, court_name, region_name):
__court = self.graph.merge_one("Court", "name", court_name)
__court.push()
self.graph.create_unique(
Relationship(__court, "SITUATED_IN", self.region(region_name)))
return __court
def chairman(self, chairman_name):
__chairman = self.graph.merge_one("Chairman", "name", chairman_name)
__chairman.push()
return __chairman
def decision_type(self, decision_type_name):
__decision_type = self.graph.merge_one("Court_Decision_Type", "name",
decision_type_name)
__decision_type.push()
return __decision_type
def judgement_type(self, judgement_type_name):
__judgement_type = self.graph.merge_one("Court_Judgement_Type", "name",
judgement_type_name)
__judgement_type.push()
return __judgement_type
def case(self, court_case, region_name):
__case = self.graph.merge_one("Case", "id", court_case.decision_number)
__case["reg_date"] = __timestamp__(court_case.reg_date)
__case["law_date"] = __timestamp__(court_case.law_date)
__case["link"] = court_case.link
__case["text"] = court_case.text
__case["case_number"] = court_case.case_number
self.graph.create_unique(
Relationship(__case, "RULED_BY",
self.court(court_case.court_name, region_name)))
self.graph.create_unique(
Relationship(__case, "CARRIED_BY",
self.chairman(court_case.chairman)))
self.graph.create_unique(
Relationship(__case, "OF_JUDGEMENT_TYPE",
self.judgement_type(court_case.vr_type)))
self.graph.create_unique(
Relationship(__case, "OF_DECISION_TYPE",
self.decision_type(court_case.cs_type)))
__case.push()
return __case
def change_date(self):
query = "MATCH (n:Case) WHERE NOT (n.law_date='') RETURN n LIMIT 5"
id_list = []
for n in self.graph.cypher.execute(query):
id_list.append(n[0].__str__()[2:].split(':')[0]) # getting an id
for _id in id_list:
n = self.graph.node(str(_id))
n['law_date'] = __timestamp__(n['law_date'])
n.push()
print(n)
class ApiProvider():
def __init__(self, request_data):
self._request_data = request_data
authenticate("localhost:7474", "neo4j", "1234")
# authenticate("52.27.227.159:7474", "neo4j", "1234")
self.graph = Graph(GRAPH_CONNECTION_STRNIG)
def _update_show(self, show_id):
# get the node from the graph
node_show = self.graph.node(show_id)
if node_show['updated'] == True:
return
result_dict = {}
try:
show_info_e_list = requests.get(
'http://services.tvrage.com/feeds/full_show_info.php?sid={0}'.format(node_show['id']))
result_dict = xmltodict.parse(show_info_e_list.text)
omdb_show_info = requests.get(
'http://www.omdbapi.com/?t={0}&y=&plot=full&r=json'.format(node_show['name']))
dict_omdb_show_info = json.loads(omdb_show_info.text)
if dict_omdb_show_info['Response'] == 'True':
for key, value in dict_omdb_show_info.iteritems():
node_show[key] = value
success = False
except ValueError as e:
logger.exception("Value Error")
return
except Exception as e:
logger.exception("Some network issue, will try again")
return
# add the new extracted data to the show
node_show['started'] = result_dict['Show'].get('started', None)
node_show['ended'] = result_dict['Show'].get('ended', None)
node_show['image'] = result_dict['Show'].get('image', None)
node_show['status'] = result_dict['Show'].get('status', None)
node_show.push()
# Country
from_country = result_dict['Show'].get('origin_country', 'unknown')
node_country = self.graph.merge_one("Country", 'country', from_country)
node_country.push()
# add the relation to the graph
show_from_country = Relationship(node_show, "from", node_country)
self.graph.create(show_from_country)
# Genres
if result_dict['Show'].get('genres', None) is not None:
genre_list = []
if type(result_dict['Show']['genres']['genre']) is list:
genre_list = result_dict['Show']['genres']['genre']
else:
genre_list.append(result_dict['Show']['genres']['genre'])
for genre in genre_list:
# create the genre node
node_genre = self.graph.merge_one("Genre", 'name', genre)
node_genre.push()
# add the Genre relation to the graph
show_of_genre = Relationship(node_show, "of genre", node_genre)
self.graph.create(show_of_genre)
# Seasons
season_list = []
if result_dict['Show'].get('Episodelist', None) is None:
return
if type(result_dict['Show']['Episodelist']['Season']) is list:
season_list = result_dict['Show']['Episodelist']['Season']
else:
season_list.append(result_dict['Show']['Episodelist']['Season'])
for season in season_list:
# create node for season
node_season = Node.cast('Season', {'no': season['@no']})
self.graph.create(node_season)
# create the relation n the graph
show_season = Relationship(node_show, "has", node_season)
self.graph.create(show_season)
# Episodes
episode_list = []
if type(season['episode']) is list:
episode_list = season['episode']
else:
episode_list.append(season['episode'])
count = 1
for episode in episode_list:
# create a node for episode
node_episode = Node.cast('Episode', {
'airdate': episode.get('airdate', None),
'epnum': count,
'screencap': episode.get('screencap', None),
'title': episode.get('title', None)
})
self.graph.create(node_episode)
# add relation to the graph
show_episode = Relationship(show_season, "has", node_episode)
self.graph.create(show_episode)
# Episode info
try:
omdb_episode_info = requests.get('http://www.omdbapi.com/?t={0}&Season={1}&Episode={2}'
.format(node_show['name'],
node_season['no'],
node_episode['epnum']))
dict_omdb_episode_info = json.loads(omdb_episode_info.text)
if dict_omdb_episode_info['Response'] == 'True':
for key, value in dict_omdb_episode_info.iteritems():
node_episode[key] = value
node_episode.push()
except ValueError as e:
logger.exception("Value error")
except Exception as e:
logger.exception("network issue: wil try again")
# links
try:
search = node_show['name'] + ' s' + str(node_season['no']).zfill(2) + 'e' + str(
node_episode['epnum']).zfill(2)
# links
search_numbers = [3552639851, 8556419051, 2649486255, 7079685853, 8416818254, 1870757059,
1731156253, 4545021852, 6021755051, 8975221455]
for n in search_numbers:
links_from_google = requests.get(
'https://www.googleapis.com/customsearch/v1element?key=AIzaSyCVAXiUzRYsML1Pv6RwSG1gunmMikTzQqY&rsz=small&num=10&hl=en&prettyPrint=false&source=gcsc&gss=.com&sig=cb6ef4de1f03dde8c26c6d526f8a1f35&cx=partner-pub-2526982841387487:{1}'
'&q={0}&googlehost=www.google.com&oq={0}'.format(search, n))
dict_from_google = json.loads(links_from_google.text)
for result in dict_from_google['results']:
# create node for link
node_link = Node.cast('Link', {
'host': result.get('visibleUrl', None),
'url': result['url']
})
self.graph.create(node_link)
# create the relation in the graph
link_episode = Relationship(node_episode, "has", node_link)
self.graph.create(link_episode)
except Exception, err:
logger.exception("error grom google part")
count = count + 1
# notify that all went OK and finish
node_show['updated'] = True
node_show.push()
