def getTestedNeo4jDB(graphDBurl, graphDbCredentials): '''Gets a Neo4j url and returns a GraphDatabaseService to the database after having performed some trivial tests''' try: if graphDbCredentials: authenticate(*graphDbCredentials) graphDb = Graph(graphDBurl) #just fetch a Node to check we are connected #even in DRY RUN we should check Neo4j connectivity #but not in OFFLINE MODE if not OFFLINE_MODE: _ = iter(graphDb.match(limit=1)).next() except StopIteration: pass except SocketError as ex: raise DbNotFoundException( ex, "Could not connect to Graph DB %s." % graphDBurl) if not DRY_RUN and not OFFLINE_MODE: try: test_node = Node("TEST", data="whatever") graphDb.create(test_node) graphDb.delete(test_node) except Exception as ex: raise DBInsufficientPrivileges(\ "Failed on trivial operations in DB %s." % graphDBurl) return graphDb
class Graph(object): def __init__(self, neo4j_uri): self.graph = NeoGraph(neo4j_uri) def find_node(self, label, node_id): args = dict(property_key="node_id", property_value=node_id) return self.graph.find_one(label, **args) def create_user(self, args): node = self.find_node("User", args["username"]) if not node: properties = dict( node_id=args["username"], name=args["name"], city=args["city"] ) node = Node("User", **properties) self.graph.create(node) return node, True return node, False def delete_user(self, user): node = self.find_node("User", user) if node: self.graph.delete(node) return True return False
def getTestedNeo4jDB(graphDBurl, graphDbCredentials): '''Gets a Neo4j url and returns a GraphDatabaseService to the database after having performed some trivial tests''' try: if graphDbCredentials: authenticate(*graphDbCredentials) graphDb = Graph(graphDBurl) #just fetch a Node to check we are connected #even in DRY RUN we should check Neo4j connectivity #but not in OFFLINE MODE if not OFFLINE_MODE: _ = iter(graphDb.match(limit=1)).next() except StopIteration: pass except SocketError as ex: raise DbNotFoundException(ex, "Could not connect to Graph DB %s." % graphDBurl) if not DRY_RUN and not OFFLINE_MODE: try: test_node = Node("TEST", data="whatever") graphDb.create(test_node) graphDb.delete(test_node) except Exception as ex: raise DBInsufficientPrivileges(\ "Failed on trivial operations in DB %s." % graphDBurl) return graphDb
def change(): if request.method == 'POST': graph = Graph('http://localhost:7474', username='******', password='******') user_change1 = request.form.get("change_company") user_change2 = request.form.get("change_position") user_change3 = request.form.get("change_name") user_change4 = request.form.get("new_name") # user_changeposition = Position.match(graph, user_change2).first() # user_changenode1 = Shareholder.match(graph, user_changerel1).first() # user_changenode2 = Company.match(graph, user_changerel2).first() matcher = NodeMatcher(graph) company_node = matcher.match("Company", name=user_change1).first() position_node = matcher.match("Position", name=user_change2, company=user_change1).first() name_node = matcher.match("name", name=user_change3).first() graph.delete(name_node) new_node = Node("name", name=user_change4) new_rel = Relationship(new_node, "属于", position_node) total_change = new_rel | new_node graph.create(total_change) ''' # 单节点对应属性修改 if len(user_change1) > 0: if user_change2 == "height": user_changenode.height = user_change3 if user_change2 == "age": user_changenode.age = user_change3 graph.push(user_changenode) # 节点关系修改 if len(user_changerel1) > 0: if user_changerel3 == "经营": user_changenode1.manager.remove(user_changenode2) graph.push(user_changenode1) relationship = Relationship(x1, user_changerel4, x2) graph.create(relationship) if user_changerel3 == "股东": user_changenode1.shareholder.remove(user_changenode2) graph.push(user_changenode1) relationship = Relationship(x1, user_changerel4, x2) graph.create(relationship) ''' return render_template('change.html')
class IntegrationTestCase(TestCase): @staticmethod def unique_string_generator(): while True: yield "_" + uuid4().hex def __init__(self, *args, **kwargs): super(IntegrationTestCase, self).__init__(*args, **kwargs) self.graph = Graph() self.node_matcher = NodeMatcher(self.graph) self.db = self.graph.database self.schema = self.graph.schema self.unique_string = self.unique_string_generator() def reset(self): graph = self.graph schema = self.schema for label in schema.node_labels: for property_keys in schema.get_uniqueness_constraints(label): schema.drop_uniqueness_constraint(label, *property_keys) for property_keys in schema.get_indexes(label): schema.drop_index(label, *property_keys) graph.delete_all() def assert_error(self, error, classes, fullname): for cls in classes: assert isinstance(error, cls) name = fullname.rpartition(".")[-1] self.assertEqual(error.__class__.__name__, error.exception, name) self.assertIn(error.fullname, [None, fullname]) self.assertTrue(error.stacktrace) def assert_new_error(self, error, classes, code): for cls in classes: assert isinstance(error, cls) name = code.rpartition(".")[-1] self.assertEqual(error.__class__.__name__, name) self.assertEqual(error.code, code) self.assertTrue(error.message) def get_non_existent_node_id(self): node = Node() self.graph.create(node) node_id = node.identity self.graph.delete(node) return node_id def get_attached_node_id(self): return self.graph.evaluate("CREATE (a)-[:TO]->(b) RETURN id(a)")
def connect(login_, password_, host_port_): # global graph allows to get the same graph every time init_db() is called authenticate(host_port_, login_, password_) if host_port_ == "localhost:7474": graph_con = Graph() else: graph_con = Graph("http://"+host_port_+"/db/data/") node = Node("TestNodeToRemove", name="Alice", age="23", eyes="blue") try: graph_con.create(node) graph_con.delete(node) except Unauthorized: graph_con = None return graph_con
def delete(): if request.method == 'POST': graph = Graph('http://localhost:7474', username='******', password='******') user_delete = request.form.get("delete_name") # 需要删除的单节点的反馈节点 user_delnode = name.match(graph, user_delete).first() if len(user_delete) > 0: graph.delete(user_delnode) ''' if len(user_del1) > 0: if user_del3 == "股东": user_delnode1.shareholder.remove(user_delnode2) graph.push(user_delnode1) if user_del3 == "经营": user_delnode.manager.remove(user_delnode2) graph.push(user_delnode1) ''' return render_template('delete.html')
def before_all(context): # import falcon_test # context.attachment_dir = os.path.join(os.path.dirname(falcon_test.__file__), 'tests/data') # context.sms_path = os.path.join(os.path.dirname(falcon_test.__file__), '../../var/sms/') # context.mail_path = os.path.join(os.path.dirname(falcon_test.__file__), '../../var/mail/') # clear database graph_db = Graph(settings.DATABASE_URL) # graph_db.delete_all() new_user_node = graph_db.find_one('USER', property_key='email', property_value='*****@*****.**') graph_db.delete(new_user_node) interest_node = graph_db.find_one('INTEREST', property_key='name', property_value=PERSONAS['interest']['name']) interest_relationships = graph_db.match(start_node=None, rel_type='INTERESTED_IN', end_node=interest_node) for relationship in interest_relationships: graph_db.delete(relationship) graph_db.delete(interest_node) context.base_url = "http://localhost:8000" benv.before_all(context)
book.icon_uri = icon_uri book.tags = tags book.bookmark.add(obj) graph.push(book) if __name__ == '__main__': graph.delete_all() bookmarks = bookmarks_parser.parse( "/home/andriy/PycharmProjects/bookmarks2db/chrome_bookmarks.html") save2db(bookmarks) # bookmark remove bookmark = Bookmark.match(graph).where( "_.url = 'https://github.com/'").first() graph.delete(bookmark) # remove folder with all nested bookmarks and folders def delete_folder(folder): for bookmark in folder.bookmarks: print('delete {} bookmark from {} folder'.format( bookmark.title, folder.title)) graph.delete(bookmark) for f in folder.folders: if f.folders or f.bookmarks: delete_folder(f) graph.delete(folder) bookmarks_bar = Folder.match(graph, "Bookmarks bar").first() delete_folder(bookmarks_bar)
print() node1['age'] = 99 graph.push(node1) data5 = graph.find(label='PersonTest') for data in data5: print("data5 = ", data) print() ''' 5 —— 删除某node,在删除node之前需要先删除relationship ''' node = graph.find_one(label='PersonTest', property_key='name', property_value="李四") relationship = graph.match_one(rel_type='KNOWNS') graph.delete(relationship) graph.delete(node) data6 = graph.find(label='PersonTest') for data in data6: print("data6 = ", data) ''' 6 —— 多条件查询 ''' a = Node('PersonTest', name='张三', age=21, location='广州') b = Node('PersonTest', name='李四', age=22, location='上海') c = Node('PersonTest', name='王五', age=21, location='北京') r1 = Relationship(a, 'KNOWS', b) r2 = Relationship(b, 'KNOWS', c) s = a | b | c | r1 | r2 graph.create(s) data7 = graph.find(label='PersonTest')
class GraphModule(NaoModule): # db path TODO make this a parameter db_path = "http://*****:*****@127.0.0.1:7474/db/data/" db = None valid_relations = ["is", "can", "similar", "describes", "likes", "dislikes"] def __init__(self, name): NaoModule.__init__(self, name) self.graph = Graph(self.db_path) def exit(self): NaoModule.exit(self) def add_constraints(self): self.add_constraint("Concept", "name") self.add_constraint("Trait", "name") self.add_constraint("Action", "name") def add_constraint(self, label, property): self.__add_constraint(label, property) def __add_constraint(self, label,property): try: self.graph.schema.create_uniqueness_constraint(label, property) print("%s:%s constraint created" % (label,property)) except: print("%s:%s constraint already exists" % (label,property)) # --------------------------- Add / Remove Nodes and Relationships ------------------- # # -- Assign a thing to a parent thing (or remove) def try_parse_relationship(self, text): params = map(lambda p: p.trim()), text.lower().split("--") if len(params) < 3: print("invalid relation use the form 'rel -- concept -- concept'") return None if self.valid_relations.count(params[0]) == 0: print("invalid relationship should be one of " + ', '.join(self.valid_relations)) return None return params def relate_concepts(self, concept1, concept2, rel, label="Concept"): if self.valid_relations.count(rel) == 0: print("invalid relationship should be one of " + ', '.join(self.valid_relations)) self.add_relationship(label, concept1, rel, label, concept2) def unrelate_concepts(self, text, label="Concept"): params = self.try_parse_relationship(text) if params is not None: rel, concept1, concept2 = params self.remove_relationship(label, concept1, rel, label, concept2) # -- Add a node (or remove) def add_concept(self, name, label="Concept"): return self.graph.cypher.execute_one("Merge(n:%s {name: '%s'}) return n" % (label, name)) # -- Add a relationship to two nodes def add_relationship(self, label1, name1, rel, label2, name2): """ add a relationship""" if rel == "can": label2 = label2 + ":" + "Action" elif rel == "describes": label1 = label1 + ":" + "Trait" n = self.graph.cypher.execute_one("Merge(n:%s {name: '%s'}) return n" % (label1, name1)) r = self.graph.cypher.execute_one("Merge(n:%s {name: '%s'}) return n" % (label2, name2)) return self.graph.create_unique(Relationship(n, rel, r)) # -- remove a relationship from two nodes def remove_relationship(self, label1, name1, rel, label2, name2): r = Relationship(Node(label1, "name", name1), rel, Node(label2, "name", name2)) self.graph.delete(r) # ---------------------------------------------------------------------------- # Query nodes for relationships and abilities # ---------------------------------------------------------------------------- # -- get definition of a thing by finding its parents def what_is(self, name, neighbors=3): stmt = "MATCH (n:Concept { name: '%s' })-[:is*1..%s]->(neighbors) RETURN neighbors.name as name" % (name, neighbors) return map(lambda x: x.name.encode('utf-8'), self.graph.cypher.execute(stmt)) # -- ask whether a thing is another thing by inheritance def is_it(self, name, parent): statement = "match p=shortestPath((a:Concept {name:'%s'})-[:IS_A*1..2]->(b:Concept {name:'%s'})) return p" % (name, parent) return self.graph.cypher.execute_one(statement) is not None # -- show examples of a thing through its children def instances_of(self, name): """ Get instances of (children of) a concept """ stmt = "MATCH (n:Concept { name: '%s' })<-[:is*1..2]-(neighbors) RETURN neighbors.name as name" % name return map(lambda x: x.name.encode('utf-8'), self.graph.cypher.execute(stmt)) # -- what can a thing do? def what_can_it_do(self, name): """ what can a concept do """ stmt = "MATCH (n:Concept { name: '%s' })-[:can]->(neighbors) RETURN neighbors.name as name" % name return map(lambda result: result.name.encode('utf-8'), self.graph.cypher.execute(stmt)) # -- can a thing perform an action (through direct knowledge) or through deduction def can_it(self, thing, action): """ can concept do concept """ stmt1 = "match p=shortestPath((a:Concept {name:'%s'})-[:can*1..3]->(b:Concept {name:'%s'})) return p" % (thing, action) if self.graph.cypher.execute_one(stmt1) is not None: return True, None stmt2 = "MATCH (n:Concept { name: '%s' })-[:IS_A*1..3]->(neighbors)-[:CAN]-(b:Concept {name: '%s'}) RETURN neighbors.name" % (thing, action) result = self.graph.cypher.execute_one(stmt2) if result is not None: return True, result.encode('utf-8') else: return False, None
# 创建节点(均可直接赋值变量) a = Node('User', name='yaim') #第1个参数为节点类型,第2个参数为节点属性和值 b = Node('User', name='fyl') # 创建关系 r = Relationship(a, 'SAME', b) # 将节点和关系加入到数据库 s = a | b | r graph.create(s) #查询节点 print(graph.nodes[0]) #根据节点id,返回节点 print(graph.nodes.get(0)) #同上 print(list(graph.nodes.match('User'))) #根据条件,返回节点列表 print(graph.nodes.match('User', name='yaim').first()) #根据条件,返回第1个节点 # 查询关系 rel_matcher = RelationshipMatcher(graph) rel_all = list(rel_matcher.match()) #获取所有关系,返回列表 rel_this = list(rel_matcher.match(r_type='SAME')) #获取某一关系,返回列表 # 删除节点 node_del = graph.nodes.match('User', name='temp').first() #先查询到某一个节点 graph.delete(node_del) #再删除 # ***删除关系 rel_del = rel_matcher.match(r_type='SAME').first() #先查询到某一个关系 graph.delete(rel_del) #再删除,*连带删除节点?* # 更新节点 node_update = graph.nodes.match('User', name='yaim').first() #先查询到某一个节点 node_update['name'] = 'yaim_new' #更新该节点某一属性值 graph.push(node_update) #提交更新 # ***更新关系
def delete(cls, pk, graph: Graph): graph.delete(cls.get(pk, graph))
class GraphDB: class SimpleNode: def __init__(self, node): self.name = node["name"].encode("utf-8") labels = map(lambda l: l.encode("utf-8"), node.labels) self.label = filter(lambda l: l != "Concept", labels)[0] db_path = "http://*****:*****@127.0.0.1:7474/db/data/" graph = None interactions = ["Request", "Engagement", "Exploration", "Judgement", "Share", "Disengagment", "Unknown"] sentiments = ["Positive", "Neutral", "Negative", "Unknown"] def __init__(self): self.graph = Graph(self.db_path) def add_constraints(self): self.add_constraint("Thing", "name") self.add_constraint("Trait", "name") self.add_constraint("Sense", "name") self.add_constraint("Action", "name") def add_index(self, label, property): self.__add_index(label, property) def __add_index(self, label, property): try: self.graph.cypher.execute(("CREATE INDEX ON :%s(%s)") % (label, property)) print("%s:%s index created" % (label, property)) except: print("%s:%s constraint already exists" % (label, property)) def add_constraint(self, label, property): self.__add_constraint(label, property) def __add_constraint(self, label, property): try: self.graph.schema.create_uniqueness_constraint(label, property) print("%s:%s constraint created" % (label, property)) except: print("%s:%s constraint already exists" % (label, property)) def add_unique_constraint(self, name, attr): self.graph.schema.create_uniqueness_constraint(name, attr) # --------------------------- Add / Remove Nodes and Relationships ------------------- # # -- Assign a thing to a parent thing (or remove) def assign_parent(self, label, name, parent, remove=False): if remove is True: return self.remove_relationship(label, name, "IS_A", label, parent) else: return self.add_relationship(label, name, "IS_A", label, parent) # -- Assign similarity between two things (or remove) def assign_similar(self, label, name, similar, remove=False): if remove is True: return self.remove_relationship(label, name, "SIMILAR_TO", label, similar) else: return self.add_relationship(label, name, "SIMILAR_TO", label, similar) # -- Assign an ability (action) to a thing (or remove) def assign_ability(self, name, a_name, remove=False): if remove is True: return self.remove_relationship("Concept", name, "CAN", "Action", a_name) else: return self.add_relationship("Thing", name, "CAN", "Action", a_name) # -- Add a node (or remove) def add_node(self, label, name, remove=False): if remove is True: return self.delete(Node(label, "name", name)) else: return self.graph.cypher.execute_one("Merge(n:Concept:%s {name: '%s'}) return n" % (label, name)) # -- Assign a trait to a thing (or remove) def assign_trait(self, thing, trait, remove=False): if remove is True: return self.remove_relationship("Trait", trait, "DESCRIBES", "Thing", thing) else: return self.add_relationship("Trait", trait, "DESCRIBES", "Thing", thing) # -- Assign a sense to a thing def assign_sense(self, thing, sense, remove=False): if remove is True: return self.remove_relationship("Thing", thing, "HAS", "Sense", sense) else: return self.add_relationship("Thing", thing, "HAS", "Sense", sense) # -- Add a relationship to two nodes def add_relationship(self, label, name, rel, r_label, r_name): """ Add a relationship :param a_label: :param a_name: :param rel: :param b_label: :param b_name: :return: """ n = self.graph.cypher.execute_one("Merge(n:Concept:%s {name: '%s'}) return n" % (label, name)) r = self.graph.cypher.execute_one("Merge(n:Concept:%s {name: '%s'}) return n" % (r_label, r_name)) return self.graph.create_unique(Relationship(n, rel, r)) # -- remove a relationship from two nodes def remove_relationship(self, label, name, rel, r_label, r_name): r = Relationship(Node(label, "name", name), rel, Node(r_label, "name", r_name)) self.graph.delete(r) # ---------------------------------------------------------------------------- # Query nodes for relationships and abilities # ---------------------------------------------------------------------------- # -- get definition of a thing by finding its parents def definition_of(self, name, neighbors=3): stmt = "MATCH (n:Concept { name: '%s' })-[:IS_A*1..%s]->(neighbors) RETURN neighbors.name as name" % ( name, neighbors, ) return map(lambda x: x.name.encode("utf-8"), self.graph.cypher.execute(stmt)) # -- ask whether a thing is another thing by inheritance def is_it_a(self, name, parent): statement = "match p=shortestPath((a:Thing {name:'%s'})-[:IS_A*1..2]->(b:Thing {name:'%s'})) return p" % ( name, parent, ) return self.graph.cypher.execute_one(statement) is not None # return self.definition_of(name).count(parent) > 0 # -- show examples of a thing through its children def examples_of(self, name): stmt = "MATCH (n:Concept { name: '%s' })<-[:IS_A*1..2]-(neighbors) RETURN neighbors.name as name" % name return map(lambda x: x.name.encode("utf-8"), self.graph.cypher.execute(stmt)) # -- what can a thing do? def what_can_it_do(self, name): """ What is a thing :param name: :param neighbors: :return: [SimpleNode(label,name)] """ stmt = "MATCH (n:Concept { name: '%s' })-[:CAN]->(neighbors) RETURN neighbors" % name results = self.graph.cypher.execute(stmt) return map(lambda result: GraphDB.SimpleNode(result.neighbors), results) # -- can a thing perform an action (through direct knowledge) or through deduction def can_it(self, thing, action): """ Can a thing do an action :param thing: :param action: :return: Tuple(Bool, NameOfMatchingParent) """ statement = "match p=shortestPath((a:Thing {name:'%s'})-[:CAN*1..3]->(b:Action {name:'%s'})) return p" % ( thing, action, ) if self.graph.cypher.execute_one(statement) is not None: return True, None statement2 = ( "MATCH (n:Concept { name: '%s' })-[:IS_A*1..3]->(neighbors)-[:CAN]-(b:Action {name: '%s'}) RETURN neighbors.name" % (thing, action) ) result = self.graph.cypher.execute_one(statement2) if result is not None: return True, result.encode("utf-8") else: return False, None
a = Node("Website",site_name ="a") graph.merge(a, 'Website', 'site_name' ) aa = list(graph.nodes.match(site_name = "a"))[0] aa aa['drop']=["drop"] aa graph.push(aa) graph.nodes.match(site_name = "a").first() aaa=list(graph.nodes.match(site_name = "a"))[0] aa['drop'].append('test') aa['drop'] graph.push(aaa) graph.nodes.match(site_name = "a").first() graph.delete(aaa) # TUTO : This creates a Website node, change it to Twitter node b = Node("Website",site_name ="b") graph.merge(b, 'Website', 'site_name' ) bb = graph.nodes.match(site_name = "b").first() bb bb.labels bb.add_label("Twitter") bb.labels bb.remove_label("Website") bb.labels graph.push(bb) graph.nodes.match(site_name = "b").first()
from py2neo import Graph, Node, Relationship test_graph = Graph( "http://localhost:7474", username="******", password="******" ) node1 = test_graph.nodes.match("animal", code="b122f437-de89-4503-8bbc-e4f4eec2762a").first() test_graph.delete(node1)
graph.push(n) if 'complete' not in n : n['complete'] = False graph.push(n) elif args[0] == "del" : if len(args) > 1 and args[1] == "all" : sys.stderr.write("Are you sure? (y/n)\n") resp=prompt() if (resp == "y") : graph.delete_all() elif len(args) > 1 : n = int(args[1])-1 if n < len(nodes) : print("Deleting Task('%s')" %(nodes[n]['title'])) graph.delete(nodes[n]) elif args[0] == "dep" : if len(args) > 1 : tgt = int(args[1])-1 for dep in args[2:] : n = int(dep)-1 if n < len(nodes) and tgt < len(nodes): print("Task('%s') <- Task('%s')" %(nodes[tgt]['title'], nodes[n]['title'])) a_dep_b = Relationship(nodes[tgt], "depends_on", nodes[n]) graph.create(a_dep_b) elif args[0] == "done" : if len(args) > 1 : id = int(args[1])
class User(object): def __init__(self, graph_db=None): self.name = '' self.call_sign = '' self.first_name = '' self.last_name = '' self.id = '' self.mission_statement = '' self.about = '' self.email = '' self.is_mentor = False self.is_tutor = False self.is_visible = True self.is_available_for_in_person = True # self._interests_list = None # self.is_admin = False # self.password = '' # self.salt = '' # self.permanent_web_token = '' # self.temporary_web_token = '' self.join_date = None self.last_active_date = '' self._graph_db = Graph(settings.DATABASE_URL) @property def user_properties(self): """ setup user properties :return: dictionary of properties """ properties_dict = dict(self.__dict__) del properties_dict['_graph_db'] return properties_dict def set_user_properties(self, user_properties): """ :param user_properties: :return: """ for key, value in user_properties.iteritems(): setattr(self, key, value) def get_user(self): user_node = self.user_node if user_node is not None: user_properties = dict(user_node.properties) for key, value in user_properties.iteritems(): setattr(self, key, value) return True else: return False def create_user(self, user_properties=None): """ create a new user based on the attributes :return: node """ #TODO exception handling self.join_date = datetime.date.today() self.last_active_date = self.join_date self.id = str(uuid.uuid4()) if user_properties is not None: self.set_user_properties(user_properties) new_user_node = Node.cast(GraphLabel.USER, self.user_properties) try: self._graph_db.create(new_user_node) except: pass # print 'node probably found. see message' # print sys.exc_info() return new_user_node @property def user_node(self): """ get a user Node :return: py2neo Node """ if self.email != '': return self._graph_db.find_one(GraphLabel.USER, property_key='email', property_value=self.email) elif self.id != '': return self._graph_db.find_one(GraphLabel.USER, property_key='id', property_value=self.id) # return self.graph_db.get_or_create_indexed_node(index_name=GraphLabel.USER, # key='email', value=self.email) @property def user_interests(self): """ get user interests :return: dictionary of interests """ user_interests = self._graph_db.match(start_node=self.user_node, rel_type=GraphRelationship.INTERESTED_IN, end_node=None) #create a list of tuples of interests and the users's relationship to them interests_list = [] for rel in user_interests: interest_dict = dict(rel.end_node.properties, **rel.properties) interests_list.append(dict(rel.end_node.properties)) return interests_list @property def user_goals(self): """ get user interests :return: list of interests """ #TODO do not need a list of interests -- HATEOAS -- MMD 3/8/2015 user_goals = self._graph_db.match(start_node=self.user_node, rel_type=GraphRelationship.HAS_GOAL, end_node=None) goals_list = [] goal_interests_list = [] for rel in user_goals: goal_properties = dict(rel.end_node.properties) goal = Goal() goal.id = goal_properties['id'] interests = goal.goal_interests interests_list = [] for interest in interests: interests_list.append(interest['name']) goal_properties['interests'] = interests_list goals_list.append(goal_properties) return goals_list @property def user_groups(self): """ :return: list of tuples of the groups """ #TODO add list of related interests user_groups = self._graph_db.match(start_node=self.user_node, rel_type=GraphRelationship.STUDIES_WITH, end_node=None) # create a list of tuples of interests and the users's relationship to them groups_list = [] for rel in user_groups: group_properties = dict(rel.end_node.properties) group = Group() group.id = group_properties['id'] interests = group.group_interests group_interests_list = [] for interest in interests: group_interests_list.append(interest['name']) group_properties['interests'] = group_interests_list groups_list.append(group_properties) return groups_list @property def user_orgs(self): """ :return: """ user_orgs = self._graph_db.match(start_node=self.user_node, rel_type=GraphRelationship.MEMBER_OF, end_node=None) orgs_list = [] for rel in user_orgs: org_properties = dict(rel.end_node.properties) org = Organization() org.id = org_properties['id'] interests = org.org_interests interests_list = [] for interest in interests: interests_list.append(interest['name']) org_properties['interests'] = interests_list orgs_list.append(org_properties) return orgs_list @property def user_locations(self): """ :return: """ user_locations = self._graph_db.match(start_node=self.user_node, rel_type=GraphRelationship.LOCATED_IN, end_node=None) locations_list = [] for rel in user_locations: locations_list.append(rel.end_node.properties) return locations_list def get_local_users_shared_interests_near_location(self): #, location_node): """ get a dictionary of user with shared interests with this user :param : :return: dictionary of {interests: [users]} """ # users_shared_interests params = { 'email': '*****@*****.**' } cypher_str = "MATCH (u:USER {email:{email}})-[url:LOCATED_IN]->(l:LOCATION)" cypher_str += "<-[orl:LOCATED_IN]-(o:USER) " cypher_str += "WITH u, o, l, url, orl " cypher_str += "MATCH (u)-[ru:INTERESTED_IN]->" cypher_str += "(i:INTEREST)<-[ro:INTERESTED_IN]-(o) " cypher_str += "RETURN i.name as interest_name, i.id as interest_id, " \ "o.name as user_name, o.id as user_id" #, u, ru, ro, l, url, orl" # print cypher_str results = Graph().cypher.execute(cypher_str, params) # self.graph_db.cypher.stream(cypher_str) # self.graph_db.cypher.execute(cypher_str) interest_users_dict = {} print results for item in results: interest = item['interest_name'] user = item['user_name'] if interest_users_dict.has_key(interest): interest_users_dict[interest].append(user) else: interest_users_dict[interest] = [] interest_users_dict[interest].append(user) # user = item['user_name'] # cur_users_list.append(interest_users_dict.get(interest)) # if interest_users_dict.has_key(interest): # # if interest in interest_users_dict.keys(): # cur_users_list = interest_users_dict[interest] # # cur_users_list = interest_users_dict.get(interest) # else: # interest_users_dict[interest] = [] # cur_users_list.append(user) # interest_users_dict[interest] = cur_users_list # interest_users_dict[interest] = interest_users_dict.get(interest) # user_details = (user_node['name'], user_node['email'], user_node['id']) # user_list.append(user_details) return interest_users_dict def add_interest(self, interest_id, experience_properties_dict=None): """ Add interest to user :param interest id:string uuid :return: List of interests """ #TODO add exception handling interest = Interest() interest.id = interest_id interest_node = interest.interest_node_by_id user_interest_relationship = Relationship(self.user_node, GraphRelationship.INTERESTED_IN, interest_node) for key, value in experience_properties_dict.iteritems(): user_interest_relationship[key] = value try: self._graph_db.create_unique(user_interest_relationship) except: pass return self.user_interests def update_interest(self, interest_id, experience_properties_dict): interest = Interest() interest.id = interest_id interest_node = interest.interest_node_by_id user_interest_relationship = self._graph_db.match_one(start_node=self.user_node, rel_type=GraphRelationship.INTERESTED_IN, end_node=interest_node) for key, value in experience_properties_dict.iteritems(): user_interest_relationship.properties[key] = value user_interest_relationship.push() def delete_interest(self, interest_id): """ drop interest relationship from user given the interest_id :param interest_id: str(uuid.uuid4()) :return: """ #TODO exception handling interest = Interest() interest.id = interest_id interest_node = interest.interest_node_by_id user_interest_relationship = self._graph_db.match_one(start_node=self.user_node, rel_type=GraphRelationship.INTERESTED_IN, end_node=interest_node) self._graph_db.delete(user_interest_relationship) def update_user(self): user_node = self.user_node user_properties = dict(self.user_properties) for key, value in user_properties.iteritems(): user_node[key] = value # user_properties[key] user_node.push() # def make_admin(self): # #new_user = self.graph_db.get_or_create_indexed_node(index_name=GraphLabel.USER, key='email', value=self.email) # self.user_node.add_labels(GraphLabel.ADMIN) def add_goal(self, goal_properties): """ Add goal to user :param goal_id: string uuid :return: List of user goals """ #TODO exception handling goal = Goal() goal.set_goal_properties(goal_properties=goal_properties) goal.create_goal() # create relationship between user and interest node user_goal_relationship = Relationship(self.user_node, GraphRelationship.HAS_GOAL, goal.goal_node) self._graph_db.create_unique(user_goal_relationship) #TODO set properties on the relationship -- may use a unique id as the key return self.user_goals def delete_goal(self, goal_id): user_node = self.user_node goal = Goal() goal.id = goal_id # have to remove all relationships before deleteing a node goal.delete_all_interests() goal_node = goal.goal_node user_goal_rel = self._graph_db.match_one(start_node=user_node, rel_type=GraphRelationship.HAS_GOAL, end_node=goal_node) self._graph_db.delete(user_goal_rel) self._graph_db.delete(goal_node) def join_group(self, group_id, group_relationship_properties=None): """ Add user as member of group :param group_id: string uuid :return: """ #TODO exception handling group = Group() group.id = group_id # relationship properties join_properties = { 'join_date': datetime.date.today() } user_group_relationship = Relationship(self.user_node, GraphRelationship.STUDIES_WITH, group.group_node) # properties=join_properties) for key, value in join_properties.iteritems(): user_group_relationship[key] = value try: self._graph_db.create_unique(user_group_relationship) except: pass #TODO set properties on the relationsip # group_relationship_properties["id"] = str(uuid.uuid4()) def leave_group(self, group_id): """ remove relationship between user and study group :param group_id: string uuid :return: None """ #TODO exception handling group = Group() group.id = group_id user_group_relationship = self._graph_db.match_one(start_node=self.user_node, rel_type=GraphRelationship.MEMBER_OF, end_node=group.group_node) self._graph_db.delete(user_group_relationship) def delete_group(self, group_id): pass def join_organization(self, organization_id): """ add user to organization :param organization_id: string uuid :return: list of tuple of interests """ #TODO exception handling org = Organization() org.id = organization_id user_org_relationship = Relationship(self.user_node, GraphRelationship.MEMBER_OF, org.org_node) try: self._graph_db.create_unique(user_org_relationship) except: print sys.exc_info()[0] def leave_organization(self, organization_id): """ remove relationship between user and organization :param organization_id: :return: """ #TODO exception handling org = Organization() org.id = organization_id user_org_relationship = self._graph_db.match_one(start_node=self.user_node, rel_type=GraphRelationship.MEMBER_OF, end_node=org.org_node) self._graph_db.delete(user_org_relationship) def add_location(self, location_json): """ link user to location nodes :param locations_place_id: :return: """ #TODO exception handling #TODO do in location and pass in the node from the actual object (better pattern) location_place_id = location_json['id'] location = Location() location.id = location_place_id location_node = location.location_node_by_place_id if not location_node: location.set_location_properties(location_json) location.create_location() location_node = location.location_node_by_place_id() user_location_relationship = Relationship(self.user_node, GraphRelationship.LOCATED_IN, location_node) # try: self._graph_db.create_unique(user_location_relationship) # except: # pass def create_cq(self, cq_dict, cq_interests_dict=None): Cq.create_cq(user_node=self.user_node, cq_dict=cq_dict) def create_converation_between_users(self, user_id_started, user_id_with, conversation_properties): # self.id = uuid.uuid4() conversation_properties['id'] = str(uuid.uuid4()) new_convo_node = Node.cast(GraphLabel.CONVERSATION, conversation_properties) try: convo_node, = self._graph_db.create(new_convo_node) # create new conversation node user_started = User() user_started.id = user_id_started user_with = User() user_with.id = user_id_with # create started conversation relationship user_started_relationship = Relationship(user_started.user_node, GraphRelationship.STARTED, convo_node) self._graph_db.create(user_started_relationship) # create started conversation with relationship convo_with_relationship = Relationship(convo_node, GraphRelationship.WITH, user_with.user_node) self._graph_db.create(convo_with_relationship) return convo_node.properties['id'] except: pass #TODO add exception handling # @staticmethod def matched_users(self, match_string, limit): """ :param match_string: :param limit: :return: dictionary of search results """ params = { 'match': '(?i)%s.*' % match_string, 'limit': limit } cypher_str = "MATCH (user:USER ) " \ "WHERE user.name =~ {match} " \ "RETURN user.name as name, user.id as id " \ "LIMIT {limit}" match_results = self._graph_db.cypher.execute(statement=cypher_str, parameters=params) root = {} root['count'] = 0 user_found = {} users_list = [] for item in match_results: user_found['id'] = item.id user_found['name'] = item.name # self.id = item['id'] # self.get_user() # users_list.append(dict(self.user_properties)) users_list.append(dict(user_found)) root['count'] += 1 root['users'] = users_list return root def register_user(self, email): verification_email = notifications.Notifications() verification_email.recipients = [email] s = URLSafeTimedSerializer(secret_key=settings.TOKEN_SECRET_KEY) payload = s.dumps(email) verification_email.subject = settings.ACTIVATION_SUBJECT verification_email.message = settings.ACTIVATION_MESSAGE verification_email.url = self.construct_verification_url(payload=payload) verification_email.send_by_gmail() def activate_user(self, payload, email): s = URLSafeTimedSerializer(secret_key=settings.TOKEN_SECRET_KEY) payload_email = s.loads(payload, max_age=settings.TOKEN_EXPIRES_IN) # 10 minutes if email == payload_email: self.email = email self.get_user() self.permanent_web_token = self.create_web_token() if self.id == '': self.create_user() else: self.update_user() else: raise BadSignature('bad email') def update_last_active_date(self): self.last_active_date = datetime.date.today() user_node = self.user_node user_node['last_active_date'] = self.last_active_date user_node.push() def construct_verification_url(self, payload): return settings.SITE_URL + settings.ACTIVATION_ROUTE + "/%s" % payload def create_web_token(self): s = URLSafeSerializer(secret_key=settings.TOKEN_SECRET_KEY) return s.dumps(self.id) def user_relationships_for_json(self, auth_id): root = self.user_profile_for_json() root['__class'] = self.__class__.__name__ root['interests'] = self.user_interests root['locations'] = self.user_locations root['goals'] = self.user_goals root['groups'] = self.user_groups root['organizations'] = self.user_orgs root['is_owner'] = (auth_id == self.id) root['allow_edit'] = (auth_id == self.id) root['allow_message'] = (auth_id is not None) return root def user_profile_for_json(self): root = self.user_properties return root def user_interests_for_json(self): root = {} root['__class'] = self.__class__.__name__ root['id'] = self.id root['email'] = self.email root['interests'] = self.user_interests return root def user_goals_for_json(self): root = {} root['__class'] = self.__class__.__name__ root['id'] = self.id root['email'] = self.email root['goals'] = self.user_goals # root['interests'] = self.user_goals['interests'] return root def user_groups_for_json(self): root = {} root['__class'] = self.__class__.__name__ root['id'] = self.id root['email'] = self.email root['groups'] = self.user_groups return root def user_locations_for_json(self, auth_id): root = {} root['__class'] = self.__class__.__name__ root['id'] = self.id if self.id == auth_id: root ['allow_edit'] = True return root def local_users_with_shared_interests_for_json(self): root = {} root['__class'] = self.__class__.__name__ root['id'] = self.id root['email'] = self.email root['users'] = self.get_local_users_shared_interests_near_location() return root def activated_user_for_json(self): root = {} root['__class'] = self.__class__.__name__ root['x_auth_key'] = self.permanent_web_token return root
class GraphdbClient(object): @LazyProperty def __db_host__(self): return self.db_host @LazyProperty def __db_port__(self): return self.db_port @LazyProperty def __db_user__(self): return self.db_user @LazyProperty def __db_password__(self): return self.db_password def get_db(self): return self.graph def __init__(self, db_host='localhost', db_port=27017, db_user='', db_password=''): self._setDatabase(db_host, db_port, db_user, db_password) def _setDatabase(self, db_host, db_port, db_user, db_password): self.db_host = db_host self.db_port = db_port self.db_user = db_user self.db_password = db_password self.graph = Graph(host=self.db_host, bolt=True, http=self.db_port, \ user=self.db_user, password=self.db_password) self.log = LogHandler("graphdb_client", level=20) def insert_or_update_node(self, label, key, vals={}): """ Insert if the node is not in the dataset or update the existed relationship. Parameters: label: label; key: key; properties: json string. Example: insert_or_update_node("Test", "yexuliang", {"Test":"123123", "has_crawled" : 1}) """ nd = Node(label, neo_id="{}_{}".format(label.lower(), key)) self.graph.merge(nd) for k, v in vals.iteritems(): val = list() if isinstance(v, list): if isinstance(v[0], dict): for elem in v: val.append(json.dumps(elem)) else: val = v nd[k] = val self.graph.push(nd) return nd def insert_or_update_relation(self, rel_type, st_nd, end_nd, vals={}): """ Insert if the relationship is not in the dataset or update the existed relationship. Parameters: rel_type: type of relationship; st_nd: start node; end_nd: end node; bidirectional: True or False; vals: json string. Example: insert_or_update_relation("T_Rel", Node("Test", uid="yexuliang"), Node("Test", uid="3310858"), {"clue":["id", "screen_name"]}) """ rel = Relationship(st_nd, rel_type, end_nd) self.graph.merge(rel) for k, v in vals.iteritems(): rel[k] = v self.graph.push(rel) def find_node_by_id(self, label, key): """ Find nodes by neo_id. Parameters: label: label; key: neo_id. Example: find_node_by_id("Douban", "Weibo_3513921") """ nds = self.graph.data( "MATCH (nd:{} {{neo_id: '{}'}}) RETURN nd".format(label, key)) if nds: return nds[0]['nd'] else: return None def find_node_by_rel(self, nd_label, nd_info, rel_type, rel_info={}, is_count=False, limit=None, skip=None): """ Find node by conditions. Parameters: label: label; nd_info: dictionary where key is the property_name and value is the property_val; rel_info: the filter of aligned info is_count: if False, return nodes; else return num of queried results limit: default None. limited number of query results. skip: for return results by segment Example: find_node_by_rel("Douban", {"name":"=~'ta'"}, 'ALIGN', {'ID': '>.9'}) """ cql = "MATCH (nd:{})-[a:{}]-(res)".format(nd_label, rel_type) nd_condition_clause = condition_clause_format('nd', nd_info) rel_cond_clause = condition_clause_format('a', rel_info) condition_clause = nd_condition_clause if rel_cond_clause: condition_clause += " AND " + rel_cond_clause if is_count: ret_clause = " RETURN count(nd) as num" else: ret_clause = " RETURN res" if condition_clause: cql += " WHERE " + condition_clause + ret_clause else: cql += ret_clause if skip and isinstance(skip, int): cql += " SKIP {}".format(skip) if limit and isinstance(limit, int): cql += " LIMIT {}".format(limit) self.log.debug(u"Query: %s" % cql) nds = self.graph.run(cql) if is_count: return nds graph_nds = list() for nd in nds: graph_nds.append(nd['res']) if graph_nds: return graph_nds else: return None def find_node_by_property(self, label, nd_info=dict(), is_count=False, limit=None, skip=None): """ Find node by conditions. Parameters: label: label; nd_info: dictionary where key is the property_name and value is the property_val; is_count: if False, return nodes; else return num of queried results limit: default None. limited number of query results. skip: for return results by segment Example: find_node_by_property("Douban", {"name":"=~'ta'"}, False) """ cql = "MATCH (nd:{})".format(label) condition_clause = condition_clause_format('nd', nd_info) if is_count: ret_clause = " RETURN count(nd) as num" else: ret_clause = " RETURN nd" if condition_clause: cql += " WHERE " + condition_clause + ret_clause else: cql += ret_clause if skip and isinstance(skip, int): cql += " SKIP {}".format(skip) if limit and isinstance(limit, int): cql += " LIMIT {}".format(limit) self.log.debug(u"Query: %s" % cql) nds = self.graph.data(cql) if is_count: return nds graph_nds = list() for nd in nds: graph_nds.append(nd['nd']) if graph_nds: return graph_nds else: return None # @deprecated(find_node_by_property) # def find_node_by_fuzzy_property(self, label, nd_info, limit=None): # """ # Find node by fuzzy conditions. # Parameters: # label: label; # nd_info: dictionary where key is the property_name and value is the property_val; # limit: default None. limited number of query results. # Example: find_node_by_fuzzy_property("Douban", {"name":"ta"}, 1) # """ # cql = "MATCH (nd:{})".format(label) # condition_clause = fuzzy_condition_clause_format('nd', nd_info) # if condition_clause: # cql += " WHERE "+condition_clause+" RETURN nd" # else: # cql += " RETURN nd" # if limit: # cql += " LIMIT {}".format(limit) # self.log.info(u"Query: %s"%cql) # nds = self.graph.data(cql) # if nds: # return nds # else: # return None # @deprecated(find_node_by_property) # def find_node_by_acc_property(self, label, nd_info, limit=None): # """ # Find node by accurate conditions. # Parameters: # label: label; # nd_info: dictionary where key is the property_name and value is the property_val; # limit: default None. limited number of query results. # Example: find_rel_by_acc_property('id', {'label':'Douban', 'name':'tada'},\ # {'label':'Weibo', 'nick_name':'tadamiracle'}) # """ # cql = "MATCH (nd:{})".format(label) # condition_clause = acc_condition_clause_format('nd', nd_info) # if condition_clause: # cql += " WHERE "+condition_clause+" RETURN nd" # else: # cql += " RETURN nd" # if limit: # cql += " LIMIT {}".format(limit) # self.log.info(u"Query: %s"%cql) # nds = self.graph.data(cql) # if nds: # return nds # else: # return None def find_rel_by_property(self, st_nd, end_nd, rel_type='ALIGN', rel_info=dict(), is_count=False, limit=None, skip=None): """ Find relationship by conditions. Parameters: st_nd: start node info in dictionary where key is the property_name and value is the property_val. end_nd: end node info in dictionary where key is the property_name and value is the property_val. rel_type: the type of relationship rel_info: the filter of relationship info is_count: if False, return relationships; else return count(rel) limit: default None. limited number of query results skip: for return results by segment Example: find_rel_by_property({'label':'Douban', 'name':"=~'.*ta.*'"}, {'label':'Weibo'}) """ if 'label' in st_nd and 'label' in end_nd: cql = "MATCH (st_nd:{})-[rel:{}]-(end_nd:{})".format( st_nd['label'], rel_type, end_nd['label']) else: self.log.warning( u"No specific relation type in 'find_rel_by_property'") return None st_cond_clause = condition_clause_format('st_nd', st_nd) end_cond_clause = condition_clause_format('end_nd', end_nd) rel_cond_clause = condition_clause_format('rel', rel_info) condition_clause = st_cond_clause if end_cond_clause: condition_clause += " AND " + end_cond_clause if rel_info: condition_clause += " AND " + rel_info if is_count: ret_clause = " RETURN count(rel) as num" else: ret_clause = " RETURN st_nd, rel, end_nd" if condition_clause: cql += " WHERE " + condition_clause + ret_clause else: cql += ret_clause if skip and isinstance(skip, int): cql += " SKIP {}".format(skip) if limit and isinstance(limit, int): cql += " LIMIT {}".format(limit) self.log.debug(u"Query: %s" % cql) rels = self.graph.data(cql) graph_rels = list() for rel in rels: graph_rels.append(rel['rel']) if graph_rels: return graph_rels else: return None # @deprecated(find_rel_by_property) # def find_rel_by_fuzzy_property(self, rel_type, st_nd, end_nd, rel_info, limit=None): # """ # Find relationship by fuzzy conditions. # Parameters: # rel_type: relation type; # st_nd: start node info in dictionary where key is the property_name and value is the property_val. # end_nd: end node info in dictionary where key is the property_name and value is the property_val. # limit: default None. limited number of query results. # Example: find_rel_by_fuzzy_property('id', {'label':'Douban', 'name':'ta'}, {'label':'Weibo'}, 2) # """ # if 'label' in st_nd and 'label' in end_nd: # cql = "MATCH (st_nd:{})-[a:ALIGN]-(end_nd:{})".format(st_nd['label'], end_nd['label']) # else: # self.log.warning(u"No specific relation type in 'find_rel_by_fuzzy_property'") # return None # st_cond_clause = fuzzy_condition_clause_format('st_nd', st_nd) # end_cond_clause = fuzzy_condition_clause_format('end_nd', end_nd) # condition_clause = st_cond_clause # if end_cond_clause: # condition_clause += " AND "+end_cond_clause # if rel_type: # condition_clause += " AND '{}' in a.align_msg".format(rel_type) # if condition_clause: # cql += " WHERE "+condition_clause+" RETURN st_nd, a, end_nd" # else: # cql += " RETURN st_nd, a, end_nd" # if limit: # cql += " LIMIT {}".format(limit) # self.log.info(u"Query: %s"%cql) # rels = self.graph.data(cql) # if rels: # return rels # else: # return None # @deprecated(find_rel_by_property) # def find_rel_by_acc_property(self, rel_type, st_nd, end_nd, limit=None): # """ # Find relationship by accurate conditions. # Parameters: # rel_type: relation type; # st_nd: start node info in dictionary where key is the property_name and value is the property_val. # end_nd: end node info in dictionary where key is the property_name and value is the property_val. # limit: default None. limited number of query results. # Example: find_rel_by_fuzzy_property('id', {'label':'Douban', 'name':'ta'}, {'label':'Weibo'}, 2) # """ # if 'label' in st_nd and 'label' in end_nd: # cql = "MATCH (st_nd:{})-[a:ALIGN]->(end_nd:{})".format(st_nd['label'], end_nd['label']) # else: # self.log.warning(u"No specific relation type in 'find_rel_by_acc_property'") # return None # st_cond_clause = acc_condition_clause_format('st_nd', st_nd) # end_cond_clause = acc_condition_clause_format('end_nd', end_nd) # condition_clause = st_cond_clause # if end_cond_clause: # condition_clause += " AND "+end_cond_clause # if rel_type: # condition_clause += " AND '{}' in a.align_msg".format(rel_type) # cql += " WHERE "+condition_clause+" RETURN st_nd, a, end_nd" # if limit: # cql += " LIMIT {}".format(limit) # self.log.info(u"Query: %s"%cql) # rels = self.graph.data(cql) # if rels: # return rels # else: # return None def delete(self, subgraph): if isinstance(subgraph, Relationship): self.graph.separate(subgraph) else: self.graph.delete(subgraph) def clear(self): self.graph.delete_all()
# has_contrast = models.Relationship(Contrast,rel_type='HASCONTRAST') for row in conditions.iterrows(): name = row[1].condition_text user = row[1].id_user uid = row[1].id task = row[1].id_term description = row[1].condition_description timestamp = row[1].event_stamp if not str(name) =="nan": properties = {"description":description} node = make_node("condition",uid,name,properties) tasknode = find_node("task",property_value=task) # If the tasknode is node found, then we do not create the relation, and we delete the condition if tasknode == None: graph.delete(node) else: relation = make_relation(tasknode,"HASCONDITION",node) #class Disorder(models.NodeModel): # has_difference = models.Relationship(Contrast,rel_type='HASDIFFERENCE') # mentioned_in = models.Relationship('PMID',rel_type='MENTIONEDIN') disorders.columns = ["id","term","classification"] # Bug in returning column index name for row in disorders.iterrows(): uid = row[1].id classification = row[1].classification name = row[1].term properties = {"classification":classification} node = make_node("disorder",uid,name,properties)
class HW3Graph(object): """ This object provides a set of helper methods for creating and retrieving nodes and relationships from a Neo4j database holding information about players, teams, fans, comments and their relationships. """ # Note: # I tend to avoid object mapping frameworks. Object mapping frameworks are fun in the beginning # but tend to be annoying after a while. So, I did not create types Player, Team, etc. # # Connects to the DB and sets a Graph instance variable. # Also creates a NodeMatcher and RelationshipMatcher, which are a py2neo framework classes. def __init__(self, auth=('neo4j', 'dbuserdbuser'), host='localhost', port=11005, secure=False, ): self._graph = Graph(secure=secure, bolt=True, auth=auth, host=host, port=port) self._node_matcher = NodeMatcher(self._graph) self._relationship_matcher = RelationshipMatcher(self._graph) def run_q(self, qs, args): """ :param qs: Query string that may have {} slots for parameters. :param args: Dictionary of parameters to insert into query string. :return: Result of the query, which executes as a single, standalone transaction. """ try: tx = self._graph.begin(autocommit=False) result = self._graph.run(qs, args) return result except Exception as e: print("Run exaception = ", e) def run_match(self, labels=None, properties=None): """ Uses a NodeMatcher to find a node matching a "template." :param labels: A list of labels that the node must have. :param properties: A dictionary of {property_name: property_value} defining the template that the node must match. :return: An array of Node objects matching the pattern. """ # ut.debug_message("Labels = ", labels) # ut.debug_message("Properties = ", json.dumps(properties)) if labels is not None and properties is not None: result = self._node_matcher.match(labels, **properties) elif labels is not None and properties is None: result = self._node_matcher.match(labels) elif labels is None and properties is not None: result = self._node_matcher.match(**properties) else: raise ValueError("Invalid request. Labels and properties cannot both be None.") # Convert NodeMatch data into a simple list of Nodes. full_result = [] for r in result: full_result.append(r) return full_result def find_nodes_by_template(self, tmp): """ :param tmp: A template defining the label and properties for Nodes to return. An example is { "label": "Fan", "template" { "last_name": "Ferguson", "first_name": "Donald" }} :return: A list of Nodes matching the template. """ labels = tmp.get('label', None) props = tmp.get("template", None) result = self.run_match(labels=labels, properties=props) return result def create_node(self, label, **kwargs) -> object: n = Node(label, **kwargs) tx = self._graph.begin(autocommit=True) tx.create(n) return n # Created method by Tim def create_relationship(self, a, r, b) -> object: g = HW3Graph() tx = g._graph.begin(autocommit=True) ab = Relationship(a, r, b) # ab is subgraph tx.create(ab) # Created method by Tim def find_by_relationship(self, template, r) -> object: key, value = list(template['template'].items())[0] tx = self._graph.begin(autocommit=True) qs = 'MATCH (n:' + template['label'] + ' {' + key + ':"' + value + '"})-[:' + r + \ ']->(node) RETURN node LIMIT 100' res = tx.run(cypher=qs) return res.data() # Created method by Tim def delete_node(self, tmp): """ :param tmp: A template defining the label and properties for Nodes to return. An example is { "label": "Fan", "template" { "last_name": "Ferguson", "first_name": "Donald" }} :return: A list of Nodes matching the template. """ flag = False subgraph = self.find_nodes_by_template(tmp) if subgraph: self._graph.delete(subgraph) flag = True return flag def update(self, old_tmp, new_tmp): """ :param tmp: A template defining the label and properties for Nodes to return. An example is { "label": "Fan", "template" { "last_name": "Ferguson", "first_name": "Donald" }} :return: A list of Nodes matching the template. """ flag = self.delete_node(old_tmp) result = None if flag: result = self.create_node(new_tmp) return result
class GetData(object): def __init__(self, uri, user, pwd, curTxBlockNum): self.graph = Graph(uri, username=user, password=pwd) self.matcher_node = NodeMatcher(self.graph) self.matcher_relation = RelationshipMatcher(self.graph) self.curTxBlockNum = curTxBlockNum # self.graph.delete_all() self.jsCode = ''' var hashjs = require('hash.js'); function getAddress(pubKey) { return hashjs.sha256().update(pubKey, 'hex').digest('hex').slice(24); } ''' self.eval = execjs.compile(self.jsCode) # 输出至控制台以及log 文件中 def printf(self, message): # print(message) with open("../log/%s.log" % time.strftime('%Y-%m-%d',time.localtime(time.time())), "a") as file: file.write(message + '\r') def getAddress(self, address, type_): if type_ == "pubKey": old_address = self.eval.call("getAddress", address[2:]) account = Account(address=old_address) return (old_address, account.bech32_address) elif type_ == "toBech32": if is_bech32_address(address): # no need to transfer to bech32 format return (address, address) account = Account(address=address) return (address, account.bech32_address) def getResult(self, method, params): data["method"] = method data["params"] = [params] try: res = requests.post("https://api.zilliqa.com/", json=data, timeout=10).json() return res["result"] if "result" in res else res except requests.exceptions.RequestException as e: self.printf("ERROR: Api Connect Timeout: Recall Api!") return self.getResult(method, params) def getTxBlcokData(self, startBlock, endBlock): for i in range(startBlock, endBlock+1): self.getOneTxBlcokData(str(i)) def getOneTxBlcokData(self, blockNum): self.printf("the %s TxBlock %s" % (blockNum, time.strftime("%y-%m-%d %H:%M:%S", time.localtime()))) # 若该Block存在,则删除 txBlock = self.matcher_node.match("TxBlock", BlockNum=blockNum).first() if txBlock: self.printf("TxBlock \t%s existed!" % blockNum) self.graph.delete(txBlock) res = self.getResult("GetTxBlock", blockNum) while "header" not in res or res["header"]["Timestamp"] == "0": # 表示到达最新的一个块,等一分钟再继续更新 self.printf("Waiting for Next Block %s" % time.strftime("%y-%m-%d %H:%M:%S", time.localtime())) time.sleep(60) res = self.getResult("GetTxBlock", blockNum) # 创建TxBlock节点以及与所属DSBlock之间的从属关系 TxBlock = Node("TxBlock") TxBlock.update({"HeaderSign": res["body"]["HeaderSign"]}) TxBlock.update(res["header"]) # 由于一个TxBlock块由多个MicroBlock组成 for index, block in enumerate(res["body"]["MicroBlockInfos"]): TxBlock.update({"MicroBlock%dHash" % index: block["MicroBlockHash"], "MicroBlock%dTxnRootHash" % index: block["MicroBlockTxnRootHash"]}) DsBlock = self.matcher_node.match("DsBlock", BlockNum=res["header"]["DSBlockNum"]).first() # 更新当前TxBlock 所对应的DsBlock if not DsBlock: DsBlock = self.getOneDsBlcokData(res["header"]["DSBlockNum"]) Tx2Ds = Relationship(TxBlock, "in", DsBlock) Ds2Tx = Relationship(DsBlock, "has", TxBlock) ########################################################## # 为has 和in 属性添加排序信息,即TxBlock 是DSBlock 中的第几个 # cur_number = len(self.matcher_relation.match([DsBlock], "has")) order = int(blockNum) % 100 Tx2Ds["order"] = order+1 Ds2Tx["order"] = order+1 # 创建Miner节点及其与TxBlock节点之间的挖与被挖关系 miner_addr_old, miner_addr_new = self.getAddress(res["header"]["MinerPubKey"], type_="pubKey") Miner, existed = self.getAccountData(miner_addr_old, miner_addr_new) Miner2Tx = Relationship(Miner, "mine", TxBlock) Tx2Miner = Relationship(TxBlock, "mined", Miner) self.graph.create(TxBlock) self.graph.create(Tx2Ds) self.graph.create(Ds2Tx) if not existed: self.graph.create(Miner) self.graph.create(Miner2Tx) self.graph.create(Tx2Miner) # 当TxBlock中有交易时,获取交易信息以及交易双方的账户信息并存入Neo4j中 if res["header"]["NumTxns"]: self.printf("%d Txs in all" % res["header"]["NumTxns"]) self.getTxData(res["header"]["BlockNum"]) def getDsBlockData(self, startBlock, endBlock): for i in range(startBlock, endBlock+1): self.getOneDsBlcokData(str(i)) # 返回DsBlock 节点 def getOneDsBlcokData(self, blockNum): self.printf("the %s DsBlock %s" % (blockNum, time.strftime("%y-%m-%d %H:%M:%S", time.localtime()))) res = self.getResult("GetDsBlock", blockNum) # 创建DSBlock节点 DsBlock = Node("DsBlock") DsBlock.update(res["header"]) DsBlock.update({"signature": res["signature"]}) # 创建Miner节点及其与DsBlock节点之间的挖与被挖关系 miner_addr_old, miner_addr_new = self.getAddress(res["header"]["LeaderPubKey"], type_="pubKey") Miner, existed = self.getAccountData(miner_addr_old, miner_addr_new) Miner2Ds = Relationship(Miner, "mine", DsBlock) Ds2Miner = Relationship(DsBlock, "mined", Miner) self.graph.create(DsBlock) if not existed: self.graph.create(Miner) self.graph.create(Miner2Ds) self.graph.create(Ds2Miner) return self.matcher_node.match("DsBlock", BlockNum=blockNum).first() def getTxData(self, txBlockNum): res, order = self.getResult("GetTransactionsForTxBlock", txBlockNum), 1 while "error" in res: self.printf("ERROR: get microblock failed! recatch txs!" + res["error"]["message"]) res = self.getResult("GetTransactionsForTxBlock", txBlockNum) for MicroBlock in res: if not MicroBlock: continue for txHash in MicroBlock: self.getOneTxData(txBlockNum, txHash, order) order += 1 # 获取一条交易的信息 @func_set_timeout(60) def getOneTxData(self, txBlockNum, txHash, order): self.printf("The %s transaction %s" % (txHash, time.strftime("%y-%m-%d %H:%M:%S", time.localtime()))) Tx = self.matcher_node.match("Tx", ID=txHash).first() if Tx: self.printf("Tx %s existed!" % txHash) # Tx 已存在时,仍需建立TxBlock 与Tx 之间的关系 TxBlock = self.matcher_node.match("TxBlock", BlockNum=txBlockNum).first() Tx2TxBlock = Relationship(Tx, "in", TxBlock) TxBlock2Tx = Relationship(TxBlock, "has", Tx) ########################################################## # 为has 和in 属性添加排序信息,即Tx 是TxBlock 中的第几个 Tx2TxBlock["order"] = order TxBlock2Tx["order"] = order self.graph.create(Tx2TxBlock) self.graph.create(TxBlock2Tx) return res_ = self.getResult("GetTransaction", txHash) # 获取交易信息时出错 ############################################# 待修改 if "error" in res_: self.printf("ERROR: get txn info failed!" + res_["error"]["message"]) return -1 # 解决调用api 时可能产生的问题 while "receipt" not in res_: res_ = self.getResult("GetTransaction", txHash) # receipt不符合格式 重新规整一下 res_["cumulative_gas"] = res_["receipt"]["cumulative_gas"] res_["epoch_num"] = res_["receipt"]["epoch_num"] res_["success"] = str(res_["receipt"]["success"]) res_.pop("receipt") from_addr_old, from_addr_new = self.getAddress(res_["senderPubKey"], type_="pubKey") to_addr_old, to_addr_new = self.getAddress(res_["toAddr"], type_="toBech32") # 创建Tx的节点以及与所属TxBlock之间的从属关系 Tx = Node("Tx") Tx.update(res_) # 由于res_["senderPubKey"]不是直接的地址,需要转换一下再将发送者的地址存储,新增bech32 格式地址存储 Tx.update({"fromAddrNew": from_addr_new, "toAddrNew": to_addr_new}) TxBlock = self.matcher_node.match("TxBlock", BlockNum=txBlockNum).first() Tx2TxBlock = Relationship(Tx, "in", TxBlock) TxBlock2Tx = Relationship(TxBlock, "has", Tx) ########################################################## # 为has 和in 属性添加排序信息,即Tx 是TxBlock 中的第几个 Tx2TxBlock["order"] = order TxBlock2Tx["order"] = order # 创建账户节点以及所参与Tx之间的发送接收关系,其中返回中 # 有一个标记信息,表示数据库中是否已经存在该节点 From_Account, existed_from = self.getAccountData(from_addr_old, from_addr_new) To_Account, existed_to = self.getAccountData(to_addr_old, to_addr_new) From2Tx = Relationship(From_Account, "send", Tx) Tx2From = Relationship(Tx, "from", From_Account) To2Tx = Relationship(To_Account, "receive", Tx) Tx2To = Relationship(Tx, "to", To_Account) From2To = Relationship(From_Account, "traded", To_Account) ################################################### # 之前便加过TxHash From2To['TxHash'] = txHash To2From = Relationship(To_Account, "traded", From_Account) To2From['TxHash'] = txHash self.graph.create(Tx) self.graph.create(Tx2TxBlock) self.graph.create(TxBlock2Tx) # 若没在数据库中 if not existed_from: self.graph.create(From_Account) if not existed_to: self.graph.create(To_Account) self.graph.create(From2Tx) self.graph.create(Tx2From) self.graph.create(To2Tx) self.graph.create(Tx2To) self.graph.create(From2To) self.graph.create(To2From) def getAccountData(self, address_old, address_new): # 当账户已经在数据库中时: Account = self.matcher_node.match("Account", address=address_old).first() res = self.getResult("GetBalance", address_old) if Account: # self.printf("account %s existed!" % address) if "error" in res: Account.update({"balance": "0", "existed": "false"}) else: Account.update({"balance": res["balance"], "existed": "true"}) self.graph.push(Account) return Account, 1 Account = Node("Account", address=address_old) Account.update({"address_new": address_new}) # 获取账户当前的余额 if "error" in res: Account.update({"balance": "0", "existed": "false"}) else: Account.update({"balance": res["balance"], "existed": "true"}) return Account, 0 def run(self): # 先对DSBlock进行存储 # self.getDsBlockData() # 在对TxBlock存储的过程中将所有的交易信息 # 以及交易过程中的账户信息一并存储到数据库 curTxBlock = self.matcher_node.match("TxBlock", BlockNum=str(self.curTxBlockNum)).first() if curTxBlock: self.graph.delete(curTxBlock) try: while True: self.getOneTxBlcokData(str(self.curTxBlockNum)) self.curTxBlockNum += 1 except FunctionTimedOut: self.printf("ERROR: Running Timeout: Restart!") self.run() except Exception as e: self.printf("ERROR: %s: Restart!" % str(e)) self.run()
for entity in entities: if entity in data[i]['Summary'].lower(): node = graph.evaluate('MATCH(e:Entity {name: "' + entity + '"}) RETURN e') if not node: node = graph.evaluate('CREATE(e:Entity {name: "' + entity + '"}) RETURN e') # Create relationship of story with an entity r = Relationship(node, "HAS_STORY", summary) graph.create(r) flag = True # No entity related to story, link to another story if not flag: # Delete the previous Story node graph.delete(summary) # Create a "Substory" node of the summary instead summary = Node("Substory", Topic=data[i]['topic'] , Summary=data[i]['Summary'] , Sentiment=data[i]['Sentiment'] , Date=data[i]['Date']) graph.create(summary) # Extract related stories query = "MATCH(n:Story) WHERE " for word in keyword.split(): query += "toLower(n.Summary) CONTAINS '" + word + "'" + " AND " query = query.rstrip(' AND ')
class TwitterGraph(): def __init__(self): self.graph = Graph("http://*****:*****@54.191.171.209:7474/db/data/") self.popularity_heap = [] self.reassess_popularity() def add_user(self, user): new_user = Node("User", token=user.token.session_id, user_id=user.id) return self.graph.create(new_user) def is_cached(self, screen_name): twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name) if twitter_user is not None: return True def get_RT_recommendations(self, user): recommendations = Counter() user_node = self.graph.find_one("User", 'user_id', user.id) following = user_node.match_outgoing("FOLLOWS", limit=5) for rel in following: retweets = rel.end_node.match_outgoing("RETWEETED", limit=5) for r in retweets: recommendations[r.end_node.properties['screen_name']] += 1 return [str for (str, count) in recommendations.most_common(10)] def get_generic_recommendations(self): return [screen_name for (count, screen_name) in heapq.nlargest(10, self.popularity_heap)] def reassess_popularity(self): # NOTE: expensive calculation, to be run threaded at multiples of x actions to graph or hourly/daily job all_twitter_users = self.graph.find("TwitterUser") for tu in all_twitter_users: incoming_count = sum(1 for _ in tu.match_incoming()) heapq.heappush(self.popularity_heap, (incoming_count, tu.properties['screen_name'])) def add_twitter_user(self, screen_name): twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name) if twitter_user is None: new_twitter_user = Node("TwitterUser", screen_name=screen_name) self.graph.create(new_twitter_user) def add_follow(self, screen_name, user): user_node = self.graph.find_one("User", 'user_id', user.id) if user_node is None: # this shouldn't happen, just for testing while transitioning db self.add_user(user) user_node = self.graph.find_one("User", 'user_id', user.id) twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name) if twitter_user is None: # this shouldn't happen, just for testing while transitioning db self.add_twitter_user(screen_name) twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name) follow_relationship = Relationship(user_node, "FOLLOWS", twitter_user) self.graph.create(follow_relationship) self.reassess_popularity() def remove_follow(self, screen_name, user): user_node = self.graph.find_one("User", 'user_id', user.id) if user_node is None: # this shouldn't happen, just for testing while transitioning db self.add_user(user) user_node = self.graph.find_one("User", 'user_id', user.id) twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name) if twitter_user is None: # this shouldn't happen, just for testing while transitioning db self.add_twitter_user(screen_name) twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name) follow_relationship = self.graph.match_one(user_node, "FOLLOWS", twitter_user) if follow_relationship is not None: self.graph.delete(follow_relationship) def add_retweet(self, screen_name, retweeted_screen_name): twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name) if twitter_user is None: # this shouldn't happen, just for testing while transitioning db self.add_twitter_user(screen_name) twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name) self.add_twitter_user(retweeted_screen_name) retweeted_twitter_user = self.graph.find_one("TwitterUser", 'screen_name', retweeted_screen_name) retweet = self.graph.match_one(twitter_user, "RETWEETED", retweeted_twitter_user) if retweet is None: retweet_relationship = Relationship(twitter_user, "RETWEETED", retweeted_twitter_user) retweet_relationship.properties['count'] = 1 self.graph.create(retweet_relationship) elif retweet.properties['count'] is None: # this shouldn't happen, just for testing while transitioning db retweet.properties['count'] = 1 retweet.push() else: retweet.properties['count'] = retweet.properties['count'] + 1 retweet.push()
# #START:更新节点 # node2 = graph.nodes[1733] # node2['stockID'] = '001' # node2['别称'] = '临时称呼' # print(graph.nodes[1733]) # # data = { # 'stockID': '002', # '别称': '惠州TCL' # } # node2.update(data) # graph.nodes[1733]._set_attr('别称','临时称呼') # print(graph.nodes[1733]) #END:更新节点 """ #Start:删除节点后,关系自动删除了 # node1 = graph.nodes[771] node2 = graph.nodes[1713] # relationship = graph.match_one(rel_type='KNOWS') # graph.delete(relationship) graph.delete(node2) #END:删除节点 """ """ #START:运行Neo4j命令 data = graph.run('MATCH (p:产品) RETURN p LIMIT 5') print(list(data)) #END:运行Neo4j命令 """
class GraphAccessor(): ## tools 类型 #EntityType = 'Person' ## 匹配Node使用的key名称 #MatchKey = 'name' def __init__(self): self.graph = Graph(host='localhost', port=7687, password='') self.nodeMatcher = NodeMatcher(self.graph) ''' 根据 entity_type, key, value 匹配获得对应的 Node. @return: ''' def matchNode(self, entity_type, key, value): return self.graph.nodes.match(entity_type).where( ' _.{} = "{}" '.format(key, value)) #return self.nodeMatcher.match(self.graph).where(' _.{} = "{}" '.format(key, value)) ''' 根据输入Entity获得对应的Node映射结果,返回Node对象. ''' def nodeMap(self, node): key = self.MatchKey value = node[key] node_list = self.matchNode(key, value) if node_list.__len__() > 0: ## 源Entity -> Neo4j Entity映射计算,获得对应的Entity ID return node_list.first() else: ## 如果Neo4j 暂无当前Entity,则添加新的Node return self.addNode(node) return None ''' 根据uuid查询获得对应的Node. ''' def findNode(self, uuid): this_node = self.graph.nodes.match(self.EntityType).where( ' _.{} = "{}" '.format('uuid', uuid)) if not this_node: return None if this_node.__len__() != 1: print sys.stderr << 'multiple node has same uuid.' return None return this_node.first() ''' 根据 entity_type, key, value 匹配获得对应的 Node. ''' def findNodes(self, entity_type, key, value): this_node = self.graph.nodes.match(entity_type).where( ' _.{} = "{}" '.format(key, value)) return this_node def findNodeOne(self, entity_type, key, value): this_node = self.graph.nodes.match(entity_type).where( ' _.{} = "{}" '.format(key, value)) assert len(this_node) <= 1 return this_node.first() ''' Graph添加新的node, 返回node对象。 ''' def addNode(self, node): self.graph.create(node) return self.findNode(node.__uuid__) ''' 根据开始、结束结点,查询获得对应的边,可能为空;不考虑边关系。 ''' def findRelation(self, start_node, end_node): #return self.graph.relationships.match().first() return self.graph.match_one(nodes=[start_node, end_node]) ''' 根据开始、结束结点以及边关系,查询获得对应的边,可能为空。 ''' def findRelationOne(self, start_node, end_node, r_type): #return self.graph.relationships.match().first() return self.graph.match_one(nodes=[start_node, end_node], r_type=r_type) ''' 删除NodeMatch下所有Node ''' def deleteNodes(self, nodes): try: for n in nodes: self.graph.delete(n) except TypeError: try: self.graph.delete(nodes) except TypeError: print "deleteNodes nodes type error." def displyNodes(self, nodes): for n in nodes: print json.dumps(n, ensure_ascii=False)
class Network(): def __init__(self): self.graph_instance = Graph() self.time = self.update_time(str(datetime.datetime.now())) # Updates current instance of time by system def update_time(self, time): self.time = time # Checks if node exists, if it does not exist - creates new node; else, updates node def add_node(self, link, date_last_updated, frequency): calculated_frequency = convert_frequency_to_hours(frequency) if (not self.check_node_exist(link)): # Create a new node for webpage with an initial calculated frequency n = Node(link, date_last_updated=date_last_updated, frequency=frequency, calculated_frequency=calculated_frequency, link=link) self.graph_instance.create(n) else: # Update existing fields for webpage node n = self.graph_instance.find_one(link) if (n["date_last_updated"] != ""): calculated_frequency = self._update_calculated_frequency( n["date_last_updated"], date_last_updated) n["date_last_updated"] = date_last_updated n["calculated_frequency"] = calculated_frequency n["frequency"] = frequency n.push() return n # Measures calculated frequency from subtracting previous date_last_updated to current date_last_updated (returns time in hours) def _update_calculated_frequency(self, prev_date_updated, new_date_updated): try: prev_date = datetime.datetime.strptime(prev_date_updated, "%Y-%m-%d") new_date = datetime.datetime.strptime(new_date_updated, "%Y-%m-%d") td = new_date - prev_date return td.total_seconds() // 3600 except: return -1 # If the relationship doesn't exist, create a new edge; else, update the tag def add_edge(self, node_u, node_v_link, relationship): self.add_node(node_v_link, "", "") node_v = self.get_node(node_v_link) self.graph_instance.create( Relationship(node_u, "links_to", node_v, tag=relationship)) def check_node_exist(self, link): return len(list(self.graph_instance.find(link))) != 0 def check_relationship_exist(self, node_u, node_v): return len( list( self.graph_instance.match(start_node=node_u, end_node=node_v, rel_type="links_to"))) > 0 def delete_failed_webpages(self, link): if (self.check_node_exist(link) == False): return node = self.get_node(link) self.delete_relationship(node) self.delete_incoming_relationship(node) self.graph_instance.delete(node) def delete_relationship(self, node_u): rels = list( self.graph_instance.match(rel_type="links_to", start_node=node_u, end_node=None)) for r in rels: self.graph_instance.separate(r) def delete_incoming_relationship(self, node_u): rels = list( self.graph_instance.match(rel_type="links_to", end_node=node_u, start_node=None)) for r in rels: self.graph_instance.separate(r) def get_node(self, link): return self.graph_instance.find_one(link) def get_node_information(self, link): check_node = self.graph_instance.data("MATCH (n {link: '" + link + "'} ) RETURN n") if len(check_node) == 0: return {} n = self.get_node(link) date_last_updated = n["date_last_updated"] calculated_frequency = n["calculated_frequency"] frequency = n["frequency"] node_data = {} node_data["date_last_updated"] = date_last_updated node_data["calculated_frequency"] = calculated_frequency node_data["frequency"] = frequency node_data["outlinks"] = self.get_outlinks(link) node_data["inlinks"] = self.get_inlinks(link) return node_data def get_outlinks(self, link): outlink_data = self.graph_instance.data("MATCH (n {link: '" + link + "'} )-->(node) RETURN node") outlinks = [] for o in outlink_data: outlinks.append(o["node"]["link"]) return outlinks def get_inlinks(self, link): inlink_data = self.graph_instance.data("MATCH (n {link: '" + link + "'} )<--(node) RETURN node") inlinks = [] for o in inlink_data: inlinks.append(o["node"]["link"]) return inlinks # Get adjacency matrix from Neo4j and nodes from py2neo def _to_matrix(self): nodes = list(self.graph_instance.node_selector.select()) N = len(nodes) mat = np.zeros((N, N)) # Populate the adjacency matrix for i, a in enumerate(nodes): for j, b in enumerate(nodes): # Use existing function to check for link mat[i, j] = self.check_relationship_exist(a, b) return mat #Iterate over nodes and add pagerank def update_pagerank(self): # Get all the nodes nodes = self.graph_instance.node_selector.select() # Iterate over the result of _pagerank and the nodes for pr, node in zip(self._pagerank(), nodes): # Update the node's pagerank and push back to neo4j node.update(page_rank=pr) self.graph_instance.push(node) # Simple show function to get nodes and display their pagerank def show_pagerank(self, selector=None, link=None): nodes = list(self.graph_instance.node_selector.select()) for node in nodes: if isinstance(link, str): if not list(node.labels())[0] == link: continue elif isinstance(link, (list, tuple)): if not list(node.labels())[0] in link: continue # Get the pageranks for any given list of links (or all) def get_pagerank_dict(self, links=[]): nodes = list(self.graph_instance.node_selector.select()) dct = {} for node in nodes: if isinstance(links, str): if not list(node.labels())[0] == links: continue elif isinstance(links, (list, tuple)): if not list(node.labels())[0] in links: continue dct[list(node.labels())[0]] = node.get('page_rank') return dct # Creates dictionary object with information for ranking API (including page rank) def get_ranking_data(self, links): page_ranks = self.get_pagerank_dict(links) data = {} data["webpages"] = [] for l in page_ranks.keys(): webpage_data = {} # If the node exists if (page_ranks[l] != None): n = self.get_node(l) webpage_data["pageRankValue"] = page_ranks[l] webpage_data["dateLastUpdated"] = n["date_last_updated"] webpage_data["frequency"] = n["frequency"] webpage_data["webpage"] = l else: webpage_data["pageRankValue"] = "NULL" webpage_data["dateLastUpdated"] = "" webpage_data["frequency"] = "" webpage_data["webpage"] = "" data["webpages"].append(webpage_data) return data # Perform pagerank on the adjacency matrix, using the power method def _pagerank( self, alpha=0.85, max_iter=100, # Increase this if we get the non-convergence error tol=1.0e-6, ): # Create a sparse matrix rep. of adjacency matrix mat = scipy.sparse.csr_matrix(self._to_matrix()) n, m = mat.shape # Make a sum matrix S = scipy.array(mat.sum(axis=1)).flatten() # Get non-zero rows index = scipy.where(S <> 0)[0] for i in index: # We need to normlize (divide by sum) mat[i, :] *= 1.0 / S[i] # pr = scipy.ones((n)) / n # initial guess # Get dangling nodes dangling = scipy.array(scipy.where(mat.sum(axis=1) == 0, 1.0 / n, 0)).flatten() for i in range(max_iter): prlast = pr pr = alpha * (pr * mat + scipy.dot(dangling, prlast)) + ( 1 - alpha) * prlast.sum() / n # check if we're done err = scipy.absolute(pr - prlast).sum() if err < n * tol: return pr raise Exception("pagerank failed to converge [%d iterations]" % (i + 1)) # Prioritizer def prioritizer(self, outlinks): # Get remaining time and number of inlink for ol in outlinks: if (not self.check_node_exist(ol)): outlinks.remove(ol) else: self.remaining_time(ol) self.sort_node(outlinks) new_links = sorted( outlinks, key=lambda k: (self.get_node(k)["time_remaining"], self.number_of_inlinks(k))) for ol in new_links: # Update last_crawled_time current = str(datetime.datetime.now()) node = self.get_node(ol) node["last_crawled_time"] = current node.push() return new_links # Get number of inlink def number_of_inlinks(self, outlink): node = self.get_node(outlink) return -len( list( self.graph_instance.match( rel_type="links_to", end_node=node, start_node=None))) # Updates remaining time left for a node to be crawled based on frequency def remaining_time(self, outlink): node = self.get_node(outlink) last_crawled_time = node["last_crawled_time"] if (last_crawled_time == None): node["time_remaining"] = 0 node.push() else: fmt = '%Y-%m-%d %H:%M:%S' current = str(datetime.datetime.now()) start = datetime.datetime.strptime(current[:19], fmt) end = datetime.datetime.strptime(last_crawled_time[:19], fmt) diff = (start - end).total_seconds() / 60.000 / 60.000 diff = float(node["calculated_frequency"]) - diff node["time_remaining"] = diff node.push() # Sort node and fill top 100 def sort_node(self, outlinks): num = len(outlinks) count = 0 nodes = self.graph_instance.data("MATCH (n) RETURN n") for n in nodes: if (not n["n"]["link"] in outlinks): self.remaining_time(n["n"]["link"]) nodes = self.graph_instance.data( "MATCH (n) RETURN n ORDER BY (n.time_remaining) DESC") for n in nodes: link = n["n"]["link"] if (not link in outlinks): outlinks.append(link) count = count + 1 if (count + num > 100): break # Return dictionary object of prioritized links and their priority value def prioritize_dic(self, outlinks): new_links = self.prioritizer(outlinks) data = {} data["prioritizedLinks"] = [] p_value = 1 for l in new_links: l_data = {} l_data["link"] = l l_data["priority_value"] = p_value * 10 data["prioritizedLinks"].append(l_data) p_value = p_value + 1 return data
# has_contrast = models.Relationship(Contrast,rel_type='HASCONTRAST') for row in conditions.iterrows(): name = row[1].condition_text user = row[1].id_user uid = row[1].id task = row[1].id_term description = row[1].condition_description timestamp = row[1].event_stamp if not str(name) == "nan": properties = {"description": description} node = make_node("condition", uid, name, properties) tasknode = find_node("task", property_value=task) # If the tasknode is node found, then we do not create the relation, and we delete the condition if tasknode == None: graph.delete(node) else: relation = make_relation(tasknode, "HASCONDITION", node) #class Disorder(models.NodeModel): # has_difference = models.Relationship(Contrast,rel_type='HASDIFFERENCE') # mentioned_in = models.Relationship('PMID',rel_type='MENTIONEDIN') disorders.columns = ["id", "term", "classification"] # Bug in returning column index name for row in disorders.iterrows(): uid = row[1].id classification = row[1].classification name = row[1].term properties = {"classification": classification} node = make_node("disorder", uid, name, properties)
test_node_2 = Node("人", name="小红") test_graph.create(test_node_1) test_graph.create(test_node_2) # 建立关系 node_1_call_node_2 = Relationship(test_node_1, '喜欢', test_node_2) # node_1_call_node_2['count'] = 1 node_2_call_node_1 = Relationship(test_node_2, '讨厌', test_node_1) # node_2_call_node_1['count'] = 2 test_graph.create(node_1_call_node_2) test_graph.create(node_2_call_node_1) # 删 test_node_3 = Node("人", name="小刚") test_graph.create(test_node_3) test_graph.delete(test_node_3) # 改 test_node_2['age'] = 20 test_graph.push(test_node_2) test_node_2['age'] = 22 test_graph.push(test_node_2) node_1_call_node_2['程度'] = '非常' test_graph.push(node_1_call_node_2) # 查 data1 = test_graph.run('MATCH (a:人) RETURN a') data = test_graph.run('MATCH (a:人) RETURN a').data()