def update_info_and_links(): print 'updating show info' authenticate("localhost:7474", "neo4j", "1234") graph = Graph(GRAPH_CONNECTION_STRNIG) results = graph.cypher.stream("match (s:Show) return id(s) as eid,s.id") start_id = 0 for record in results: if int(record['s.id']) < start_id: continue node_show = graph.node(record['eid']) result_dict = {} success = True while success: try: show_info_e_list = requests.get( 'http://services.tvrage.com/feeds/full_show_info.php?sid={0}'.format(node_show['id'])) result_dict = xmltodict.parse(show_info_e_list.text) omdb_show_info = requests.get( 'http://www.omdbapi.com/?t={0}&y=&plot=full&r=json'.format(node_show['name'])) dict_omdb_show_info = json.loads(omdb_show_info.text) if dict_omdb_show_info['Response'] == 'True': for key, value in dict_omdb_show_info.iteritems(): node_show[key] = value success = False except ValueError as e: logger.exception("Value error") continue except Exception as e: logger.exception("Some network issue: will try again") success = True print str(node_show['name']) # info node_show['started'] = result_dict['Show'].get('started', None) node_show['ended'] = result_dict['Show'].get('ended', None) node_show['image'] = result_dict['Show'].get('image', None) node_show['status'] = result_dict['Show'].get('status', None) node_show.push() #Country from_country = result_dict['Show'].get('origin_country', 'unknown') node_country = graph.merge_one("Country", 'country', from_country) node_country.push() show_from_country = Relationship(node_show, "from", node_country) graph.create(show_from_country) #Genres if result_dict['Show'].get('genres', None) is not None: genre_list = [] if type(result_dict['Show']['genres']['genre']) is list: genre_list = result_dict['Show']['genres']['genre'] else: genre_list.append(result_dict['Show']['genres']['genre']) for genre in genre_list: node_genre = graph.merge_one("Genre", 'name', genre) node_genre.push() show_of_genre = Relationship(node_show, "of genre", node_genre) graph.create(show_of_genre) """ try: print node_show['started'] a = node_show['started'].split("/") if int(a[len(a)-1]) < 2000: continue except Exception: continue """ #Seasons season_list = [] if result_dict['Show'].get('Episodelist', None) is None: continue if type(result_dict['Show']['Episodelist']['Season']) is list: season_list = result_dict['Show']['Episodelist']['Season'] else: season_list.append(result_dict['Show']['Episodelist']['Season']) for season in season_list: node_season = Node.cast('Season', {'no': season['@no']}) graph.create(node_season) show_season = Relationship(node_show, "has", node_season) graph.create(show_season) #Episodes episode_list = [] if type(season['episode']) is list: episode_list = season['episode'] else: episode_list.append(season['episode']) count = 1 for episode in episode_list: node_episode = Node.cast('Episode', { 'airdate': episode.get('airdate', None), 'epnum': count, 'screencap': episode.get('screencap', None), 'title': episode.get('title', None) }) graph.create(node_episode) success = True while success: try: omdb_episode_info = requests.get('http://www.omdbapi.com/?t={0}&Season={1}&Episode={2}' .format(node_show['name'], node_season['no'], node_episode['epnum'])) dict_omdb_episode_info = json.loads(omdb_episode_info.text) if dict_omdb_episode_info['Response'] == 'True': for key, value in dict_omdb_episode_info.iteritems(): node_episode[key] = value node_episode.push() success = False except ValueError as e: logger.exception("Value error") continue except Exception as e: logger.exception("Some network issue: will try again") success = True try: search = node_show['name'] + ' s' + str(node_season['no']).zfill(2) + 'e' + str(node_episode['epnum']).zfill(2) print search #links search_numbers = [3552639851, 8556419051, 2649486255, 7079685853, 8416818254, 1870757059, 1731156253, 4545021852, 6021755051, 8975221455] for n in search_numbers: links_from_google = requests.get( 'https://www.googleapis.com/customsearch/v1element?key=AIzaSyCVAXiUzRYsML1Pv6RwSG1gunmMikTzQqY&rsz=small&num=10&hl=en&prettyPrint=false&source=gcsc&gss=.com&sig=cb6ef4de1f03dde8c26c6d526f8a1f35&cx=partner-pub-2526982841387487:{1}' '&q={0}&googlehost=www.google.com&oq={0}'.format(search, n)) dict_from_google = json.loads(links_from_google.text) for result in dict_from_google['results']: node_link = Node.cast('Link', { 'host': result.get('visibleUrl', None), 'url': result['url'] }) graph.create(node_link) link_episode = Relationship(node_episode, "has", node_link) graph.create(link_episode) except Exception, err: logger.exception("error grom google part") show_episode = Relationship(show_season, "has", node_episode) graph.create(show_episode) count = count + 1
if a and b: ENDPOINT = "http://*****:*****@localhost:7474/db/data/" request = { "to":ENDPOINT+"node/"+str(b._id), "max_depth": int(sys.argv[3]), "relationships": { "type":"LINKS", "direction":"out" }, "algorithm":"allSimplePaths" } r = requests.post(ENDPOINT+"node/"+str(a._id)+"/paths", data=json.dumps(request)) # print r.json() if r.status_code == 200: for path in r.json(): print "Path:" for node in path['nodes']: # print node retrieved_node = graph.node(node.split('/node/')[1]) print retrieved_node.properties['title'], "->" print "--------------------" else: print "Something went wrong." print sys.exc_info()[0] else: if not a: print sys.argv[1], "not found!" else: print sys.argv[2], "not found!"
def update_show_info_old(): print 'updating show info' authenticate("localhost:7474", "neo4j", "1234") graph = Graph(GRAPH_CONNECTION_STRNIG) results = graph.cypher.stream("match (s:Show) return id(s) as eid,s.id") start_id = 764 for record in results: if int(record['s.id']) < start_id: continue node_show = graph.node(record['eid']) result_dict = {} success = True while success: try: show_info_e_list = requests.get( 'http://services.tvrage.com/feeds/full_show_info.php?sid={0}'.format(node_show['id'])) result_dict = xmltodict.parse(show_info_e_list.text) omdb_show_info = requests.get( 'http://www.omdbapi.com/?t={0}&y=&plot=full&r=json'.format(node_show['name'])) dict_omdb_show_info = json.loads(omdb_show_info.text) if dict_omdb_show_info['Response'] == 'True': for key, value in dict_omdb_show_info.iteritems(): node_show[key] = value success = False except ValueError as e: logger.exception("Value Error") continue except Exception as e: logger.exception("Some network issue, will try again") success = True print str(node_show['name']) # info node_show['started'] = result_dict['Show'].get('started', None) node_show['ended'] = result_dict['Show'].get('ended', None) node_show['image'] = result_dict['Show'].get('image', None) node_show['status'] = result_dict['Show'].get('status', None) node_show.push() #Country from_country = result_dict['Show'].get('origin_country', 'unknown') node_country = graph.merge_one("Country", 'country', from_country) node_country.push() show_from_country = Relationship(node_show, "from", node_country) graph.create(show_from_country) #Genres if result_dict['Show'].get('genres', None) is not None: genre_list = [] if type(result_dict['Show']['genres']['genre']) is list: genre_list = result_dict['Show']['genres']['genre'] else: genre_list.append(result_dict['Show']['genres']['genre']) for genre in genre_list: node_genre = graph.merge_one("Genre", 'name', genre) node_genre.push() show_of_genre = Relationship(node_show, "of genre", node_genre) graph.create(show_of_genre) """try: print node_show['started'] a = node_show['started'].split("/") if int(a[len(a)-1]) < 2000: continue except Exception: continue """ #Seasons season_list = [] if result_dict['Show'].get('Episodelist', None) is None: continue if type(result_dict['Show']['Episodelist']['Season']) is list: season_list = result_dict['Show']['Episodelist']['Season'] else: season_list.append(result_dict['Show']['Episodelist']['Season']) for season in season_list: node_season = Node.cast('Season', {'no': season['@no']}) graph.create(node_season) show_season = Relationship(node_show, "has", node_season) graph.create(show_season) #Episodes episode_list = [] if type(season['episode']) is list: episode_list = season['episode'] else: episode_list.append(season['episode']) count = 1 for episode in episode_list: node_episode = Node.cast('Episode', { 'airdate': episode.get('airdate', None), 'epnum': count, 'screencap': episode.get('screencap', None), 'title': episode.get('title', None) }) graph.create(node_episode) success = True while success: try: omdb_episode_info = requests.get('http://www.omdbapi.com/?t={0}&Season={1}&Episode={2}' .format(node_show['name'], node_season['no'], node_episode['epnum'])) dict_omdb_episode_info = json.loads(omdb_episode_info.text) if dict_omdb_episode_info['Response'] == 'True': for key, value in dict_omdb_episode_info.iteritems(): node_episode[key] = value node_episode.push() success = False except ValueError as e: logger.exception("Value error") continue except Exception as e: logger.exception("network issue: wil try again") success = True show_episode = Relationship(show_season, "has", node_episode) graph.create(show_episode) count = count + 1 print 'end updating show info'
r1 = floor(tot_len * (i + 1) * 0.01) if l1-1 == r1: l1 = r1 cursor1 = db.get_cursor() cursor1.execute(query1,(str(tmony),str(l1),str(r1))) if(i == 50): while True: row = cursor1.fetchone() if not row: break # Process each of these UNIQUE parent id's here if row: term_node = graph.find_one("TERM", "term_id", row["parent_id"]) segment_node_id = graph.cypher.execute(query2neo4id, {"tmony": tmony, "segment": str(i+1)}) segment_node = graph.node(segment_node_id[0][0]) has_term_relationship = Relationship(segment_node, "HAS_TERM", term_node) is_in_relationship = Relationship(term_node, "IS_IN", segment_node) graph.create(has_term_relationship) graph.create(is_in_relationship) cursor1.close() count += 1
__author__ = 'SU915198' from py2neo import Node, Rel, Graph # node_name = Node("http://localhost:7474/db/data") graph = Graph("http://localhost:7474/") n = graph.node("People")
class GraphDB: def __init__(self,username, password, server, port): ''' username = '******' password = '******' server = 'localhost' port = '7474' ''' self.username = username self.password = password self.server = server self.port = port self.con_url = 'http://'+username+':'+password+'@'+server+':'+port+'/db/data/' self.graph = Graph(self.con_url) ##TODO: move these to constants.py when IPYTHON not required self.metaprops = { 'RESOLVEDUUID':'_resolvedWithUUID_', 'RESOLVEDRELID':'_resolvedWithRELID_', 'RESOLVEDHENID':'_resolvedWithHENID_', ##for hyper edge node 'RESOLVEDHERID':'_resolvedWithHERID_', ##for hyper edge relation } #Done: Multiple each time want to ask something #Else: works on old data only ##Wont use this def getGraph(self): return self.graph ##NOT TO BE USED #this is how we use the intenal ids of the graph ##should we use it?? most people say no ## anyways, made the method for future use ## Use : getNodeByInternalId(graph, 152) def getNodeByInternalId(self,id): a = self.graph.node(id) #getting by id given internally by neo4j a.pull() return a ##NOT TO BE USED ## similar to above ## Use : getRelByInternalId(graph, 4) def getRelByInternalId(self,id): a = self.graph.relationship(id) a.pull() return a def getNodeByUniqueID(self, uniquelabel, idName, idVal, isIDString=False): ##TODO: move uuid to props query = "match (n:"+uniquelabel+" {" if isIDString: query = query+ idName+":'"+str(idVal)+"'}) return n" else: query = query+ idName+":"+str(idVal)+"}) return n" rc = self.graph.cypher.execute(query) return rc[0][0] def getRelationByUniqueID(self, idName, idVal, isIDString=False): ##TODO: move uuid to props query = "match ()-[r {" if isIDString: query = query+ idName+":'"+str(idVal)+"'}]->() return r" else: query = query+ idName+":"+str(idVal)+"}]->() return r" rc = self.graph.cypher.execute(query) return rc[0][0] def isPropList(self, node, prop): return type(node[prop]) is list ##has a counter brother in nexusapis flask app def isValidNonMetaProp(self, propname): if propname[0]=='_' or propname[-1]=='_': return False return True ##copy meta = True def copyNode(self, node, copymeta = True, exceptions = []): #exceptions are the props that should be included no matter what, if they have underscore or not! naya = Node() for label in node.labels: naya.labels.add(label) for prop in exceptions: if prop in node.properties: naya[prop] = node[prop] for prop in node.properties: if not copymeta: if self.isValidNonMetaProp(prop): naya[prop] = node[prop] else: naya[prop] = node[prop] return naya def copyNodeAsItIs(self, node): return self.copyNode(node) # naya = Node() # for label in node.labels: # naya.labels.add(label) # for prop in node.properties: # naya[prop] = node[prop] # return naya def copyNodeWithoutMeta(self, node, exceptions=[]): return self.copyNode(node, copymeta = False, exceptions = exceptions) def copyRelation(self, rel, copymeta = True, rel_exceptions = [], node_exceptions = []): start_node = '' end_node = '' if not copymeta: start_node = self.copyNodeWithoutMeta(rel.start_node, exceptions = node_exceptions) end_node = self.copyNodeWithoutMeta(rel.end_node, exceptions = node_exceptions) else: start_node = self.copyNodeAsItIs(rel.start_node) end_node = self.copyNodeAsItIs(rel.end_node) reltype = rel.type nayarel = Relationship(start_node, reltype, end_node) for prop in rel_exceptions: if prop in rel.properties: nayarel[prop] = rel[prop] for prop in rel.properties: if not copymeta: if self.isValidNonMetaProp(prop): nayarel[prop] = rel[prop] else: nayarel[prop] = rel[prop] return nayarel def copyRelationAsItIs(self, rel): return self.copyRelation(rel) # start_node = g.copyNodeAsItIs(rel.start_node) # end_node = g.copyNodeAsItIs(rel.end_node) # reltype = rel.type # naya = Relationship(start_node, reltype, end_node) # for prop in rel.properties: # naya[prop] = rel[prop] # return naya def copyObjectAsItIs(self, kind, obj): if kind == 'relation': return self.copyRelationAsItIs(obj) if kind == 'node': return self.copyNodeAsItIs(obj) return None def copyRelationWithoutMeta(self, rel, rel_exceptions = [], node_exceptions = []): return self.copyRelation(rel, copymeta = False, rel_exceptions = rel_exceptions, node_exceptions = node_exceptions) def getDirectlyConnectedEntities(self, idname, idval, uniquelabel='', isIDString = True): ''' given idname like uuid, henid and uniquelabel like entity or hyperedgenode gives us the connected nodes of this node returns nodes which have uuids only note: works for hyperedgenodes only for now #TODO: extend so that this info can be shown on view page ''' from app.constants import LABEL_ENTITY invertedComma = '' if isIDString: invertedComma = "'" query = "match (n:%s {%s:%s%s%s})-[]-(p:%s) return distinct(p)" query = query %(uniquelabel, idname, invertedComma, idval, invertedComma, LABEL_ENTITY) results = self.graph.cypher.execute(query) enlist = [] for res in results: enlist.append(res[0]) return enlist def getDirectlyConnectedRelations(self, idname, idval, uniquelabel='', isIDString = True,outgoing=True): ''' given idname like uuid, henid and uniquelabel like entity or hyperedgenode gives us the connected nodes of this node returns nodes which have uuids only note: works for hyperedgenodes only for now #TODO: extend so that this info can be shown on view page ''' from app.constants import LABEL_ENTITY invertedComma = '' if isIDString: invertedComma = "'" query = '' if outgoing: query = "match (n:%s {%s:%s%s%s})-[p]->(:%s) return distinct(p)" else: query = "match (n:%s {%s:%s%s%s})<-[p]-(:%s) return distinct(p)" query = query %(uniquelabel, idname, invertedComma, idval, invertedComma, LABEL_ENTITY) results = self.graph.cypher.execute(query) rellist = [] for res in results: rellist.append(res[0]) return rellist ##example of some text input: (n154346:businessperson:person:politician {name:"Anita",uuid:1234}) ##Usage: deserializeNode('''(n154346:businessperson:person:politician {name:"Anita",uuid:1234})''') def deserializeNode(self, nodeText): pos = nodeText.find(' ') #get the labels in a set startText = nodeText[1:pos] allLabels = startText.split(':')[1:] allLabels = set(allLabels) #set is imp #get the props in a dict endText = nodeText[pos+1:-1] endTextWB = endText[1:-1] #print endText #print endTextWB propList = endTextWB.split(",") propsDict = {} for x in propList: propval = x.split(":") #for handling the single inverted comma problem prop = propval[0] val = propval[1] if val[0]=="'" and val[-1]=="'": val=val[1:-1] #for handling the double inverted comma problem if val[0]=='"' and val[-1]=='"': val=val[1:-1] propsDict[prop]=val #print propsDict #creating the node from parsedText node = Node() for x in allLabels: node.labels.add(x) for x in propsDict: node[x] = propsDict[x] print node return node ##MAJOR TODO: ##work to do add a form where they can create a node #get a node's page by uuid #also get node's relations in form of graph, embed that graph #edit a node #edit a relation #dlete a node #delete a relation #moderate any change --> how to do that --> where will this lie! #Note the diff between now and then def generateSearchData(self, idname, idval, isIDString, getList=False): from app.utils.commonutils import Utils comp = self.getNodeByUniqueID('entity', idname, idval, isIDString) ##comp is the node in question keywords = '' labels = '' aliases_to_return = '' if getList: keywords = [] labels = [] aliases_to_return = [] quotes = "" ##the ' wre reudundant from beginning for prop in comp.properties: ##begins with underscore ignore if prop!='uuid' and prop!='aliases' and prop[0]!='_': currvalue = str(comp.properties[prop]) currvalue = Utils.processString(currvalue) if prop!='uuid' and len(currvalue)>3: if not getList: keywords = keywords + quotes +currvalue + quotes+"," else: keywords.append(currvalue) neighbours = self.getDirectlyConnectedEntities(idname, idval, 'entity', isIDString) for rel in neighbours: currvalue = str(rel.properties['name']) currvalue = Utils.processString(currvalue) if not getList: keywords = keywords + quotes +currvalue + quotes+"," else: keywords.append(rel.properties['name']) # keywords = keywords + '"' # labels = '' for label in list(comp.labels): if not getList: labels = labels + quotes + label + quotes+"," else: labels.append(label) # labels=labels + '"' aliases = [] if aliases is None or aliases == []: aliases = [comp['name']] ##patch from app.utils.commonutils import Utils aliases = Utils.merge(comp['aliases'],comp['name']) print 'allllllllllllllaaaaaaaa' print aliases # if type(aliases) is not list: ##fix for just one aliases, though wont be req # aliases = [aliases] # ##patch for alias in aliases: if not getList: aliases_to_return = aliases_to_return + quotes +str(Utils.processString(alias)) + quotes+"," else: aliases_to_return.append(alias) # aliases =aliases_to_return + '"' # name = '"'+comp.properties['name']+'"' name = comp.properties['name'] print 'aliases_to_return, aliases, aliases_to_return' print aliases_to_return print keywords if len(keywords)>50: keywords=keywords[:50] return name, labels, aliases_to_return, keywords
class DatabaseConnection(object): def __init__(self): authenticate("localhost:7474", "neo4j", "admin") self.connection = Graph() def add_person(self, person_name, year_of_birth): self.connection.create(Node("Person", name=person_name, born=year_of_birth)) def add_relationship(self, start_id, end_id, relationship_type): start_node = self.connection.node(start_id) end_node = self.connection.node(end_id) self.connection.create(Relationship(start_node, relationship_type, end_node)) def remove_relationship(self, relationship_id): self.connection.cypher.stream( "MATCH (n) - [r] - () WHERE ID(r) = {} DELETE r".format(relationship_id)) def remove_person(self, person_id): self.connection.cypher.stream( "MATCH(p:Person) where ID(p) = {} OPTIONAL MATCH(p) - [r] - () DELETE r, p".format( person_id)) def add_person_property(self): pass def update_person(self, person_id, person_properties): node_to_update = self.get_neo4j_person(person_id) keys = node_to_update.properties.keys() for key in keys: del node_to_update.properties[key] if person_properties: for key in person_properties: node_to_update.properties[key] = person_properties[key] node_to_update.push() def update_relationship(self, relationship_id, relationship_properties): relationship_to_update = self.get_neo4j_relationship(relationship_id) keys = relationship_to_update.properties.keys() for key in keys: del relationship_to_update.properties[key] if relationship_properties: for key in relationship_properties: relationship_to_update.properties[key] = relationship_properties[key] relationship_to_update.push() def get_neo4j_person(self, person_id): single_node_list = self.connection.cypher.stream( "MATCH(p:Person) where ID(p) = {} RETURN p".format(person_id)) for a_node in single_node_list: return a_node[0] def get_person(self, person_id): neo_node = self.get_neo4j_person(person_id) return person.Person(str(neo_node.uri).rsplit('/', 1)[-1], neo_node.properties["name"], neo_node.properties["born"], neo_node.properties) def get_all_persons(self): nodes = list() for n in self.connection.cypher.stream("START z=node(*) RETURN z"): new_node = person.Person(str(n[0].uri).rsplit('/', 1)[-1], n[0].properties["name"], n[0].properties["born"], n[0].properties) nodes.append(new_node) return nodes def get_relationship(self, relationship_id): neo_relationship = self.get_neo4j_relationship(relationship_id) relationship_to_return = relationship.Relationship( relationship_id=str(neo_relationship.uri).rsplit('/', 1)[-1], start_node=neo_relationship.start_node, end_node=neo_relationship.end_node, relationship_type=neo_relationship.type, properties=neo_relationship.properties) return relationship_to_return def get_neo4j_relationship(self, relationship_id): single_relationship_list = self.connection.cypher.stream( "MATCH (n) - [r] - () WHERE ID(r) = {} RETURN r" .format(relationship_id)) for a_relation in single_relationship_list: return a_relation[0] def get_all_relationships(self): relations = list() for relation in self.connection.cypher.stream("Match (a)-[r]->(b) return r"): new_relationship = relationship.Relationship( relationship_id=str(relation[0].uri).rsplit('/', 1)[-1], start_node=relation[0].start_node, end_node=relation[0].end_node, relationship_type=relation[0].type, properties=relation[0].properties) relations.append(new_relationship) return relations def get_child_generations_for_person(self, person_id): return_value = next(self.connection.cypher. stream( "MATCH p=(r:Person)<-[:CHILD_OF*1..20]-(x) WHERE ID(r) = {} RETURN max(length(p))". format(person_id)))[0] return_value = return_value if return_value else 0 return return_value def get_parent_generations_for_person(self, person_id): return_value = next(self.connection.cypher. stream( "MATCH p=(r:Person)-[:CHILD_OF*1..20]->(x) WHERE ID(r) = {} RETURN max(length(p))". format(person_id)))[0] return_value = return_value if return_value else 0 return return_value def is_descendant_of(self, descendant_person_id, ancestor_person_id): path_length = next(self.connection.cypher. stream( "MATCH p=(r:Person)-[:CHILD_OF*1..20]->(q:Person) WHERE ID(r) = {} and ID(q) = {}" "RETURN max(length(p))". format(descendant_person_id, ancestor_person_id)))[0] return True if path_length else False def is_married(self, person_id): path_length = next(self.connection.cypher. stream( "MATCH p=(r:Person)-[:MARRIED]-(x) WHERE ID(r) = {} RETURN max(length(p))". format(person_id)))[0] return True if path_length else False def get_spouses_of_person(self, person_id): spouse_list = list() for spouse in self.connection.cypher.stream( "MATCH p=(r:Person)-[:MARRIED]-(q:Person) WHERE ID(r) = {} " "RETURN q".format(person_id)): spouse_list.append(str(spouse[0].uri).rsplit('/', 1)[-1]) return spouse_list def get_parents_of_person(self, person_id): parents_list = list() for parent in self.connection.cypher.stream( "MATCH p=(r:Person)-[:CHILD_OF]->(q:Person) WHERE ID(r) = {} RETURN q". format(person_id)): parents_list.append(str(parent[0].uri).rsplit('/', 1)[-1]) return parents_list if parents_list else False def get_children_of_person(self, person_id): children_list = list() for parent in self.connection.cypher.stream( "MATCH p=(r:Person)<-[:CHILD_OF]-(q:Person) WHERE ID(r) = {} RETURN q". format(person_id)): children_list.append(str(parent[0].uri).rsplit('/', 1)[-1]) return children_list if children_list else False def get_relationships_of_a_node(self, person_id): relationship_list = list() return_value = next(self.connection.cypher.stream( "MATCH p=(person:Person)-[r]-() WHERE ID(person) = {} WITH collect(r) AS rs RETURN rs". format(person_id))) start_place = 0 while True: start_place = str(return_value).find("relationship/", start_place) + len( "relationship/") if start_place == -1 + len("relationship/"): break end_place = str(return_value).find("'", start_place) relationship_list.append( self.get_relationship(str(return_value)[start_place:end_place])) start_place = end_place return relationship_list
class Neo4jModel: def __init__(self): self.graph = Graph() def create(self): self.graph.schema.create_uniqueness_constraint("Region", "name") self.graph.schema.create_uniqueness_constraint("Court", "name") self.graph.schema.create_uniqueness_constraint("Court_Decision_Type", "name") self.graph.schema.create_uniqueness_constraint("Court_Judgement_Type", "name") self.graph.schema.create_uniqueness_constraint("Case", "id") self.graph.schema.create_uniqueness_constraint("Chairman", "name") def region(self, region_name): __region = self.graph.merge_one("Region", "name", region_name) __region.push() return __region def court(self, court_name, region_name): __court = self.graph.merge_one("Court", "name", court_name) __court.push() self.graph.create_unique(Relationship(__court, "SITUATED_IN", self.region(region_name))) return __court def chairman(self, chairman_name): __chairman = self.graph.merge_one("Chairman", "name", chairman_name) __chairman.push() return __chairman def decision_type(self, decision_type_name): __decision_type = self.graph.merge_one("Court_Decision_Type", "name", decision_type_name) __decision_type.push() return __decision_type def judgement_type(self, judgement_type_name): __judgement_type = self.graph.merge_one("Court_Judgement_Type", "name", judgement_type_name) __judgement_type.push() return __judgement_type def case(self, court_case, region_name): __case = self.graph.merge_one("Case", "id", court_case.decision_number) __case["reg_date"] = __timestamp__(court_case.reg_date) __case["law_date"] = __timestamp__(court_case.law_date) __case["link"] = court_case.link __case["text"] = court_case.text __case["case_number"] = court_case.case_number self.graph.create_unique(Relationship(__case, "RULED_BY", self.court(court_case.court_name, region_name))) self.graph.create_unique(Relationship(__case, "CARRIED_BY", self.chairman(court_case.chairman))) self.graph.create_unique(Relationship(__case, "OF_JUDGEMENT_TYPE", self.judgement_type(court_case.vr_type))) self.graph.create_unique(Relationship(__case, "OF_DECISION_TYPE", self.decision_type(court_case.cs_type))) __case.push() return __case def change_date(self): query = "MATCH (n:Case) WHERE NOT (n.law_date='') RETURN n LIMIT 5" id_list = [] for n in self.graph.cypher.execute(query): id_list.append(n[0].__str__()[2:].split(':')[0]) # getting an id for _id in id_list: n = self.graph.node(str(_id)) n['law_date'] = __timestamp__(n['law_date']) n.push() print(n)
class Neo4jModel: def __init__(self): self.graph = Graph() def create(self): self.graph.schema.create_uniqueness_constraint("Region", "name") self.graph.schema.create_uniqueness_constraint("Court", "name") self.graph.schema.create_uniqueness_constraint("Court_Decision_Type", "name") self.graph.schema.create_uniqueness_constraint("Court_Judgement_Type", "name") self.graph.schema.create_uniqueness_constraint("Case", "id") self.graph.schema.create_uniqueness_constraint("Chairman", "name") def region(self, region_name): __region = self.graph.merge_one("Region", "name", region_name) __region.push() return __region def court(self, court_name, region_name): __court = self.graph.merge_one("Court", "name", court_name) __court.push() self.graph.create_unique( Relationship(__court, "SITUATED_IN", self.region(region_name))) return __court def chairman(self, chairman_name): __chairman = self.graph.merge_one("Chairman", "name", chairman_name) __chairman.push() return __chairman def decision_type(self, decision_type_name): __decision_type = self.graph.merge_one("Court_Decision_Type", "name", decision_type_name) __decision_type.push() return __decision_type def judgement_type(self, judgement_type_name): __judgement_type = self.graph.merge_one("Court_Judgement_Type", "name", judgement_type_name) __judgement_type.push() return __judgement_type def case(self, court_case, region_name): __case = self.graph.merge_one("Case", "id", court_case.decision_number) __case["reg_date"] = __timestamp__(court_case.reg_date) __case["law_date"] = __timestamp__(court_case.law_date) __case["link"] = court_case.link __case["text"] = court_case.text __case["case_number"] = court_case.case_number self.graph.create_unique( Relationship(__case, "RULED_BY", self.court(court_case.court_name, region_name))) self.graph.create_unique( Relationship(__case, "CARRIED_BY", self.chairman(court_case.chairman))) self.graph.create_unique( Relationship(__case, "OF_JUDGEMENT_TYPE", self.judgement_type(court_case.vr_type))) self.graph.create_unique( Relationship(__case, "OF_DECISION_TYPE", self.decision_type(court_case.cs_type))) __case.push() return __case def change_date(self): query = "MATCH (n:Case) WHERE NOT (n.law_date='') RETURN n LIMIT 5" id_list = [] for n in self.graph.cypher.execute(query): id_list.append(n[0].__str__()[2:].split(':')[0]) # getting an id for _id in id_list: n = self.graph.node(str(_id)) n['law_date'] = __timestamp__(n['law_date']) n.push() print(n)
class ApiProvider(): def __init__(self, request_data): self._request_data = request_data authenticate("localhost:7474", "neo4j", "1234") # authenticate("52.27.227.159:7474", "neo4j", "1234") self.graph = Graph(GRAPH_CONNECTION_STRNIG) def _update_show(self, show_id): # get the node from the graph node_show = self.graph.node(show_id) if node_show['updated'] == True: return result_dict = {} try: show_info_e_list = requests.get( 'http://services.tvrage.com/feeds/full_show_info.php?sid={0}'.format(node_show['id'])) result_dict = xmltodict.parse(show_info_e_list.text) omdb_show_info = requests.get( 'http://www.omdbapi.com/?t={0}&y=&plot=full&r=json'.format(node_show['name'])) dict_omdb_show_info = json.loads(omdb_show_info.text) if dict_omdb_show_info['Response'] == 'True': for key, value in dict_omdb_show_info.iteritems(): node_show[key] = value success = False except ValueError as e: logger.exception("Value Error") return except Exception as e: logger.exception("Some network issue, will try again") return # add the new extracted data to the show node_show['started'] = result_dict['Show'].get('started', None) node_show['ended'] = result_dict['Show'].get('ended', None) node_show['image'] = result_dict['Show'].get('image', None) node_show['status'] = result_dict['Show'].get('status', None) node_show.push() # Country from_country = result_dict['Show'].get('origin_country', 'unknown') node_country = self.graph.merge_one("Country", 'country', from_country) node_country.push() # add the relation to the graph show_from_country = Relationship(node_show, "from", node_country) self.graph.create(show_from_country) # Genres if result_dict['Show'].get('genres', None) is not None: genre_list = [] if type(result_dict['Show']['genres']['genre']) is list: genre_list = result_dict['Show']['genres']['genre'] else: genre_list.append(result_dict['Show']['genres']['genre']) for genre in genre_list: # create the genre node node_genre = self.graph.merge_one("Genre", 'name', genre) node_genre.push() # add the Genre relation to the graph show_of_genre = Relationship(node_show, "of genre", node_genre) self.graph.create(show_of_genre) # Seasons season_list = [] if result_dict['Show'].get('Episodelist', None) is None: return if type(result_dict['Show']['Episodelist']['Season']) is list: season_list = result_dict['Show']['Episodelist']['Season'] else: season_list.append(result_dict['Show']['Episodelist']['Season']) for season in season_list: # create node for season node_season = Node.cast('Season', {'no': season['@no']}) self.graph.create(node_season) # create the relation n the graph show_season = Relationship(node_show, "has", node_season) self.graph.create(show_season) # Episodes episode_list = [] if type(season['episode']) is list: episode_list = season['episode'] else: episode_list.append(season['episode']) count = 1 for episode in episode_list: # create a node for episode node_episode = Node.cast('Episode', { 'airdate': episode.get('airdate', None), 'epnum': count, 'screencap': episode.get('screencap', None), 'title': episode.get('title', None) }) self.graph.create(node_episode) # add relation to the graph show_episode = Relationship(show_season, "has", node_episode) self.graph.create(show_episode) # Episode info try: omdb_episode_info = requests.get('http://www.omdbapi.com/?t={0}&Season={1}&Episode={2}' .format(node_show['name'], node_season['no'], node_episode['epnum'])) dict_omdb_episode_info = json.loads(omdb_episode_info.text) if dict_omdb_episode_info['Response'] == 'True': for key, value in dict_omdb_episode_info.iteritems(): node_episode[key] = value node_episode.push() except ValueError as e: logger.exception("Value error") except Exception as e: logger.exception("network issue: wil try again") # links try: search = node_show['name'] + ' s' + str(node_season['no']).zfill(2) + 'e' + str( node_episode['epnum']).zfill(2) # links search_numbers = [3552639851, 8556419051, 2649486255, 7079685853, 8416818254, 1870757059, 1731156253, 4545021852, 6021755051, 8975221455] for n in search_numbers: links_from_google = requests.get( 'https://www.googleapis.com/customsearch/v1element?key=AIzaSyCVAXiUzRYsML1Pv6RwSG1gunmMikTzQqY&rsz=small&num=10&hl=en&prettyPrint=false&source=gcsc&gss=.com&sig=cb6ef4de1f03dde8c26c6d526f8a1f35&cx=partner-pub-2526982841387487:{1}' '&q={0}&googlehost=www.google.com&oq={0}'.format(search, n)) dict_from_google = json.loads(links_from_google.text) for result in dict_from_google['results']: # create node for link node_link = Node.cast('Link', { 'host': result.get('visibleUrl', None), 'url': result['url'] }) self.graph.create(node_link) # create the relation in the graph link_episode = Relationship(node_episode, "has", node_link) self.graph.create(link_episode) except Exception, err: logger.exception("error grom google part") count = count + 1 # notify that all went OK and finish node_show['updated'] = True node_show.push()