def getTestedNeo4jDB(graphDBurl, graphDbCredentials):
'''Gets a Neo4j url and returns a GraphDatabaseService to the database
after having performed some trivial tests'''
try:
if graphDbCredentials:
authenticate(*graphDbCredentials)
graphDb = Graph(graphDBurl)
#just fetch a Node to check we are connected
#even in DRY RUN we should check Neo4j connectivity
#but not in OFFLINE MODE
if not OFFLINE_MODE:
_ = iter(graphDb.match(limit=1)).next()
except StopIteration:
pass
except SocketError as ex:
raise DbNotFoundException(
ex, "Could not connect to Graph DB %s." % graphDBurl)
if not DRY_RUN and not OFFLINE_MODE:
try:
test_node = Node("TEST", data="whatever")
graphDb.create(test_node)
graphDb.delete(test_node)
except Exception as ex:
raise DBInsufficientPrivileges(\
"Failed on trivial operations in DB %s." % graphDBurl)
return graphDb
class Graph(object):
def __init__(self, neo4j_uri):
self.graph = NeoGraph(neo4j_uri)
def find_node(self, label, node_id):
args = dict(property_key="node_id", property_value=node_id)
return self.graph.find_one(label, **args)
def create_user(self, args):
node = self.find_node("User", args["username"])
if not node:
properties = dict(
node_id=args["username"],
name=args["name"],
city=args["city"]
)
node = Node("User", **properties)
self.graph.create(node)
return node, True
return node, False
def delete_user(self, user):
node = self.find_node("User", user)
if node:
self.graph.delete(node)
return True
return False
def getTestedNeo4jDB(graphDBurl, graphDbCredentials):
'''Gets a Neo4j url and returns a GraphDatabaseService to the database
after having performed some trivial tests'''
try:
if graphDbCredentials:
authenticate(*graphDbCredentials)
graphDb = Graph(graphDBurl)
#just fetch a Node to check we are connected
#even in DRY RUN we should check Neo4j connectivity
#but not in OFFLINE MODE
if not OFFLINE_MODE:
_ = iter(graphDb.match(limit=1)).next()
except StopIteration:
pass
except SocketError as ex:
raise DbNotFoundException(ex, "Could not connect to Graph DB %s."
% graphDBurl)
if not DRY_RUN and not OFFLINE_MODE:
try:
test_node = Node("TEST", data="whatever")
graphDb.create(test_node)
graphDb.delete(test_node)
except Exception as ex:
raise DBInsufficientPrivileges(\
"Failed on trivial operations in DB %s." % graphDBurl)
return graphDb
def change():
if request.method == 'POST':
graph = Graph('http://localhost:7474', username='******', password='******')
user_change1 = request.form.get("change_company")
user_change2 = request.form.get("change_position")
user_change3 = request.form.get("change_name")
user_change4 = request.form.get("new_name")
# user_changeposition = Position.match(graph, user_change2).first()
# user_changenode1 = Shareholder.match(graph, user_changerel1).first()
# user_changenode2 = Company.match(graph, user_changerel2).first()
matcher = NodeMatcher(graph)
company_node = matcher.match("Company", name=user_change1).first()
position_node = matcher.match("Position", name=user_change2, company=user_change1).first()
name_node = matcher.match("name", name=user_change3).first()
graph.delete(name_node)
new_node = Node("name", name=user_change4)
new_rel = Relationship(new_node, "属于", position_node)
total_change = new_rel | new_node
graph.create(total_change)
'''
# 单节点对应属性修改
if len(user_change1) > 0:
if user_change2 == "height":
user_changenode.height = user_change3
if user_change2 == "age":
user_changenode.age = user_change3
graph.push(user_changenode)
# 节点关系修改
if len(user_changerel1) > 0:
if user_changerel3 == "经营":
user_changenode1.manager.remove(user_changenode2)
graph.push(user_changenode1)
relationship = Relationship(x1, user_changerel4, x2)
graph.create(relationship)
if user_changerel3 == "股东":
user_changenode1.shareholder.remove(user_changenode2)
graph.push(user_changenode1)
relationship = Relationship(x1, user_changerel4, x2)
graph.create(relationship)
'''
return render_template('change.html')
class IntegrationTestCase(TestCase):
@staticmethod
def unique_string_generator():
while True:
yield "_" + uuid4().hex
def __init__(self, *args, **kwargs):
super(IntegrationTestCase, self).__init__(*args, **kwargs)
self.graph = Graph()
self.node_matcher = NodeMatcher(self.graph)
self.db = self.graph.database
self.schema = self.graph.schema
self.unique_string = self.unique_string_generator()
def reset(self):
graph = self.graph
schema = self.schema
for label in schema.node_labels:
for property_keys in schema.get_uniqueness_constraints(label):
schema.drop_uniqueness_constraint(label, *property_keys)
for property_keys in schema.get_indexes(label):
schema.drop_index(label, *property_keys)
graph.delete_all()
def assert_error(self, error, classes, fullname):
for cls in classes:
assert isinstance(error, cls)
name = fullname.rpartition(".")[-1]
self.assertEqual(error.__class__.__name__, error.exception, name)
self.assertIn(error.fullname, [None, fullname])
self.assertTrue(error.stacktrace)
def assert_new_error(self, error, classes, code):
for cls in classes:
assert isinstance(error, cls)
name = code.rpartition(".")[-1]
self.assertEqual(error.__class__.__name__, name)
self.assertEqual(error.code, code)
self.assertTrue(error.message)
def get_non_existent_node_id(self):
node = Node()
self.graph.create(node)
node_id = node.identity
self.graph.delete(node)
return node_id
def get_attached_node_id(self):
return self.graph.evaluate("CREATE (a)-[:TO]->(b) RETURN id(a)")
def connect(login_, password_, host_port_):
# global graph allows to get the same graph every time init_db() is called
authenticate(host_port_, login_, password_)
if host_port_ == "localhost:7474":
graph_con = Graph()
else:
graph_con = Graph("http://"+host_port_+"/db/data/")
node = Node("TestNodeToRemove", name="Alice", age="23", eyes="blue")
try:
graph_con.create(node)
graph_con.delete(node)
except Unauthorized:
graph_con = None
return graph_con
def delete():
if request.method == 'POST':
graph = Graph('http://localhost:7474', username='******', password='******')
user_delete = request.form.get("delete_name") # 需要删除的单节点的反馈节点
user_delnode = name.match(graph, user_delete).first()
if len(user_delete) > 0:
graph.delete(user_delnode)
'''
if len(user_del1) > 0:
if user_del3 == "股东":
user_delnode1.shareholder.remove(user_delnode2)
graph.push(user_delnode1)
if user_del3 == "经营":
user_delnode.manager.remove(user_delnode2)
graph.push(user_delnode1)
'''
return render_template('delete.html')
def before_all(context):
# import falcon_test
# context.attachment_dir = os.path.join(os.path.dirname(falcon_test.__file__), 'tests/data')
# context.sms_path = os.path.join(os.path.dirname(falcon_test.__file__), '../../var/sms/')
# context.mail_path = os.path.join(os.path.dirname(falcon_test.__file__), '../../var/mail/')
# clear database
graph_db = Graph(settings.DATABASE_URL)
# graph_db.delete_all()
new_user_node = graph_db.find_one('USER',
property_key='email',
property_value='*****@*****.**')
graph_db.delete(new_user_node)
interest_node = graph_db.find_one('INTEREST', property_key='name',
property_value=PERSONAS['interest']['name'])
interest_relationships = graph_db.match(start_node=None,
rel_type='INTERESTED_IN',
end_node=interest_node)
for relationship in interest_relationships:
graph_db.delete(relationship)
graph_db.delete(interest_node)
context.base_url = "http://localhost:8000"
benv.before_all(context)
book.icon_uri = icon_uri
book.tags = tags
book.bookmark.add(obj)
graph.push(book)
if __name__ == '__main__':
graph.delete_all()
bookmarks = bookmarks_parser.parse(
"/home/andriy/PycharmProjects/bookmarks2db/chrome_bookmarks.html")
save2db(bookmarks)
# bookmark remove
bookmark = Bookmark.match(graph).where(
"_.url = 'https://github.com/'").first()
graph.delete(bookmark)
# remove folder with all nested bookmarks and folders
def delete_folder(folder):
for bookmark in folder.bookmarks:
print('delete {} bookmark from {} folder'.format(
bookmark.title, folder.title))
graph.delete(bookmark)
for f in folder.folders:
if f.folders or f.bookmarks:
delete_folder(f)
graph.delete(folder)
bookmarks_bar = Folder.match(graph, "Bookmarks bar").first()
delete_folder(bookmarks_bar)
print()
node1['age'] = 99
graph.push(node1)
data5 = graph.find(label='PersonTest')
for data in data5:
print("data5 = ", data)
print()
'''
5 —— 删除某node，在删除node之前需要先删除relationship
'''
node = graph.find_one(label='PersonTest',
property_key='name',
property_value="李四")
relationship = graph.match_one(rel_type='KNOWNS')
graph.delete(relationship)
graph.delete(node)
data6 = graph.find(label='PersonTest')
for data in data6:
print("data6 = ", data)
'''
6 —— 多条件查询
'''
a = Node('PersonTest', name='张三', age=21, location='广州')
b = Node('PersonTest', name='李四', age=22, location='上海')
c = Node('PersonTest', name='王五', age=21, location='北京')
r1 = Relationship(a, 'KNOWS', b)
r2 = Relationship(b, 'KNOWS', c)
s = a | b | c | r1 | r2
graph.create(s)
data7 = graph.find(label='PersonTest')
class GraphModule(NaoModule):
# db path TODO make this a parameter
db_path = "http://*****:*****@127.0.0.1:7474/db/data/"
db = None
valid_relations = ["is", "can", "similar", "describes", "likes", "dislikes"]
def __init__(self, name):
NaoModule.__init__(self, name)
self.graph = Graph(self.db_path)
def exit(self):
NaoModule.exit(self)
def add_constraints(self):
self.add_constraint("Concept", "name")
self.add_constraint("Trait", "name")
self.add_constraint("Action", "name")
def add_constraint(self, label, property):
self.__add_constraint(label, property)
def __add_constraint(self, label,property):
try:
self.graph.schema.create_uniqueness_constraint(label, property)
print("%s:%s constraint created" % (label,property))
except:
print("%s:%s constraint already exists" % (label,property))
# --------------------------- Add / Remove Nodes and Relationships ------------------- #
# -- Assign a thing to a parent thing (or remove)
def try_parse_relationship(self, text):
params = map(lambda p: p.trim()), text.lower().split("--")
if len(params) < 3:
print("invalid relation use the form 'rel -- concept -- concept'")
return None
if self.valid_relations.count(params[0]) == 0:
print("invalid relationship should be one of " + ', '.join(self.valid_relations))
return None
return params
def relate_concepts(self, concept1, concept2, rel, label="Concept"):
if self.valid_relations.count(rel) == 0:
print("invalid relationship should be one of " + ', '.join(self.valid_relations))
self.add_relationship(label, concept1, rel, label, concept2)
def unrelate_concepts(self, text, label="Concept"):
params = self.try_parse_relationship(text)
if params is not None:
rel, concept1, concept2 = params
self.remove_relationship(label, concept1, rel, label, concept2)
# -- Add a node (or remove)
def add_concept(self, name, label="Concept"):
return self.graph.cypher.execute_one("Merge(n:%s {name: '%s'}) return n" % (label, name))
# -- Add a relationship to two nodes
def add_relationship(self, label1, name1, rel, label2, name2):
""" add a relationship"""
if rel == "can":
label2 = label2 + ":" + "Action"
elif rel == "describes":
label1 = label1 + ":" + "Trait"
n = self.graph.cypher.execute_one("Merge(n:%s {name: '%s'}) return n" % (label1, name1))
r = self.graph.cypher.execute_one("Merge(n:%s {name: '%s'}) return n" % (label2, name2))
return self.graph.create_unique(Relationship(n, rel, r))
# -- remove a relationship from two nodes
def remove_relationship(self, label1, name1, rel, label2, name2):
r = Relationship(Node(label1, "name", name1), rel, Node(label2, "name", name2))
self.graph.delete(r)
# ----------------------------------------------------------------------------
# Query nodes for relationships and abilities
# ----------------------------------------------------------------------------
# -- get definition of a thing by finding its parents
def what_is(self, name, neighbors=3):
stmt = "MATCH (n:Concept { name: '%s' })-[:is*1..%s]->(neighbors) RETURN neighbors.name as name" % (name, neighbors)
return map(lambda x: x.name.encode('utf-8'), self.graph.cypher.execute(stmt))
# -- ask whether a thing is another thing by inheritance
def is_it(self, name, parent):
statement = "match p=shortestPath((a:Concept {name:'%s'})-[:IS_A*1..2]->(b:Concept {name:'%s'})) return p" % (name, parent)
return self.graph.cypher.execute_one(statement) is not None
# -- show examples of a thing through its children
def instances_of(self, name):
""" Get instances of (children of) a concept """
stmt = "MATCH (n:Concept { name: '%s' })<-[:is*1..2]-(neighbors) RETURN neighbors.name as name" % name
return map(lambda x: x.name.encode('utf-8'), self.graph.cypher.execute(stmt))
# -- what can a thing do?
def what_can_it_do(self, name):
""" what can a concept do """
stmt = "MATCH (n:Concept { name: '%s' })-[:can]->(neighbors) RETURN neighbors.name as name" % name
return map(lambda result: result.name.encode('utf-8'), self.graph.cypher.execute(stmt))
# -- can a thing perform an action (through direct knowledge) or through deduction
def can_it(self, thing, action):
""" can concept do concept """
stmt1 = "match p=shortestPath((a:Concept {name:'%s'})-[:can*1..3]->(b:Concept {name:'%s'})) return p" % (thing, action)
if self.graph.cypher.execute_one(stmt1) is not None:
return True, None
stmt2 = "MATCH (n:Concept { name: '%s' })-[:IS_A*1..3]->(neighbors)-[:CAN]-(b:Concept {name: '%s'}) RETURN neighbors.name" % (thing, action)
result = self.graph.cypher.execute_one(stmt2)
if result is not None:
return True, result.encode('utf-8')
else:
return False, None
# 创建节点（均可直接赋值变量）
a = Node('User', name='yaim') #第1个参数为节点类型，第2个参数为节点属性和值
b = Node('User', name='fyl')
# 创建关系
r = Relationship(a, 'SAME', b)
# 将节点和关系加入到数据库
s = a | b | r
graph.create(s)
#查询节点
print(graph.nodes[0]) #根据节点id，返回节点
print(graph.nodes.get(0)) #同上
print(list(graph.nodes.match('User'))) #根据条件，返回节点列表
print(graph.nodes.match('User', name='yaim').first()) #根据条件，返回第1个节点
# 查询关系
rel_matcher = RelationshipMatcher(graph)
rel_all = list(rel_matcher.match()) #获取所有关系，返回列表
rel_this = list(rel_matcher.match(r_type='SAME')) #获取某一关系，返回列表
# 删除节点
node_del = graph.nodes.match('User', name='temp').first() #先查询到某一个节点
graph.delete(node_del) #再删除
# ***删除关系
rel_del = rel_matcher.match(r_type='SAME').first() #先查询到某一个关系
graph.delete(rel_del) #再删除，*连带删除节点？*
# 更新节点
node_update = graph.nodes.match('User', name='yaim').first() #先查询到某一个节点
node_update['name'] = 'yaim_new' #更新该节点某一属性值
graph.push(node_update) #提交更新
# ***更新关系
def delete(cls, pk, graph: Graph):
graph.delete(cls.get(pk, graph))
class GraphDB:
class SimpleNode:
def __init__(self, node):
self.name = node["name"].encode("utf-8")
labels = map(lambda l: l.encode("utf-8"), node.labels)
self.label = filter(lambda l: l != "Concept", labels)[0]
db_path = "http://*****:*****@127.0.0.1:7474/db/data/"
graph = None
interactions = ["Request", "Engagement", "Exploration", "Judgement", "Share", "Disengagment", "Unknown"]
sentiments = ["Positive", "Neutral", "Negative", "Unknown"]
def __init__(self):
self.graph = Graph(self.db_path)
def add_constraints(self):
self.add_constraint("Thing", "name")
self.add_constraint("Trait", "name")
self.add_constraint("Sense", "name")
self.add_constraint("Action", "name")
def add_index(self, label, property):
self.__add_index(label, property)
def __add_index(self, label, property):
try:
self.graph.cypher.execute(("CREATE INDEX ON :%s(%s)") % (label, property))
print("%s:%s index created" % (label, property))
except:
print("%s:%s constraint already exists" % (label, property))
def add_constraint(self, label, property):
self.__add_constraint(label, property)
def __add_constraint(self, label, property):
try:
self.graph.schema.create_uniqueness_constraint(label, property)
print("%s:%s constraint created" % (label, property))
except:
print("%s:%s constraint already exists" % (label, property))
def add_unique_constraint(self, name, attr):
self.graph.schema.create_uniqueness_constraint(name, attr)
# --------------------------- Add / Remove Nodes and Relationships ------------------- #
# -- Assign a thing to a parent thing (or remove)
def assign_parent(self, label, name, parent, remove=False):
if remove is True:
return self.remove_relationship(label, name, "IS_A", label, parent)
else:
return self.add_relationship(label, name, "IS_A", label, parent)
# -- Assign similarity between two things (or remove)
def assign_similar(self, label, name, similar, remove=False):
if remove is True:
return self.remove_relationship(label, name, "SIMILAR_TO", label, similar)
else:
return self.add_relationship(label, name, "SIMILAR_TO", label, similar)
# -- Assign an ability (action) to a thing (or remove)
def assign_ability(self, name, a_name, remove=False):
if remove is True:
return self.remove_relationship("Concept", name, "CAN", "Action", a_name)
else:
return self.add_relationship("Thing", name, "CAN", "Action", a_name)
# -- Add a node (or remove)
def add_node(self, label, name, remove=False):
if remove is True:
return self.delete(Node(label, "name", name))
else:
return self.graph.cypher.execute_one("Merge(n:Concept:%s {name: '%s'}) return n" % (label, name))
# -- Assign a trait to a thing (or remove)
def assign_trait(self, thing, trait, remove=False):
if remove is True:
return self.remove_relationship("Trait", trait, "DESCRIBES", "Thing", thing)
else:
return self.add_relationship("Trait", trait, "DESCRIBES", "Thing", thing)
# -- Assign a sense to a thing
def assign_sense(self, thing, sense, remove=False):
if remove is True:
return self.remove_relationship("Thing", thing, "HAS", "Sense", sense)
else:
return self.add_relationship("Thing", thing, "HAS", "Sense", sense)
# -- Add a relationship to two nodes
def add_relationship(self, label, name, rel, r_label, r_name):
"""
Add a relationship
:param a_label:
:param a_name:
:param rel:
:param b_label:
:param b_name:
:return:
"""
n = self.graph.cypher.execute_one("Merge(n:Concept:%s {name: '%s'}) return n" % (label, name))
r = self.graph.cypher.execute_one("Merge(n:Concept:%s {name: '%s'}) return n" % (r_label, r_name))
return self.graph.create_unique(Relationship(n, rel, r))
# -- remove a relationship from two nodes
def remove_relationship(self, label, name, rel, r_label, r_name):
r = Relationship(Node(label, "name", name), rel, Node(r_label, "name", r_name))
self.graph.delete(r)
# ----------------------------------------------------------------------------
# Query nodes for relationships and abilities
# ----------------------------------------------------------------------------
# -- get definition of a thing by finding its parents
def definition_of(self, name, neighbors=3):
stmt = "MATCH (n:Concept { name: '%s' })-[:IS_A*1..%s]->(neighbors) RETURN neighbors.name as name" % (
name,
neighbors,
)
return map(lambda x: x.name.encode("utf-8"), self.graph.cypher.execute(stmt))
# -- ask whether a thing is another thing by inheritance
def is_it_a(self, name, parent):
statement = "match p=shortestPath((a:Thing {name:'%s'})-[:IS_A*1..2]->(b:Thing {name:'%s'})) return p" % (
name,
parent,
)
return self.graph.cypher.execute_one(statement) is not None
# return self.definition_of(name).count(parent) > 0
# -- show examples of a thing through its children
def examples_of(self, name):
stmt = "MATCH (n:Concept { name: '%s' })<-[:IS_A*1..2]-(neighbors) RETURN neighbors.name as name" % name
return map(lambda x: x.name.encode("utf-8"), self.graph.cypher.execute(stmt))
# -- what can a thing do?
def what_can_it_do(self, name):
"""
What is a thing
:param name:
:param neighbors:
:return: [SimpleNode(label,name)]
"""
stmt = "MATCH (n:Concept { name: '%s' })-[:CAN]->(neighbors) RETURN neighbors" % name
results = self.graph.cypher.execute(stmt)
return map(lambda result: GraphDB.SimpleNode(result.neighbors), results)
# -- can a thing perform an action (through direct knowledge) or through deduction
def can_it(self, thing, action):
"""
Can a thing do an action
:param thing:
:param action:
:return: Tuple(Bool, NameOfMatchingParent)
"""
statement = "match p=shortestPath((a:Thing {name:'%s'})-[:CAN*1..3]->(b:Action {name:'%s'})) return p" % (
thing,
action,
)
if self.graph.cypher.execute_one(statement) is not None:
return True, None
statement2 = (
"MATCH (n:Concept { name: '%s' })-[:IS_A*1..3]->(neighbors)-[:CAN]-(b:Action {name: '%s'}) RETURN neighbors.name"
% (thing, action)
)
result = self.graph.cypher.execute_one(statement2)
if result is not None:
return True, result.encode("utf-8")
else:
return False, None
a = Node("Website",site_name ="a")
graph.merge(a, 'Website', 'site_name' )
aa = list(graph.nodes.match(site_name = "a"))[0]
aa
aa['drop']=["drop"]
aa
graph.push(aa)
graph.nodes.match(site_name = "a").first()
aaa=list(graph.nodes.match(site_name = "a"))[0]
aa['drop'].append('test')
aa['drop']
graph.push(aaa)
graph.nodes.match(site_name = "a").first()
graph.delete(aaa)
# TUTO : This creates a Website node, change it to Twitter node
b = Node("Website",site_name ="b")
graph.merge(b, 'Website', 'site_name' )
bb = graph.nodes.match(site_name = "b").first()
bb
bb.labels
bb.add_label("Twitter")
bb.labels
bb.remove_label("Website")
bb.labels
graph.push(bb)
graph.nodes.match(site_name = "b").first()
from py2neo import Graph, Node, Relationship
test_graph = Graph(
"http://localhost:7474",
username="******",
password="******"
)
node1 = test_graph.nodes.match("animal", code="b122f437-de89-4503-8bbc-e4f4eec2762a").first()
test_graph.delete(node1)
graph.push(n)
if 'complete' not in n :
n['complete'] = False
graph.push(n)
elif args[0] == "del" :
if len(args) > 1 and args[1] == "all" :
sys.stderr.write("Are you sure? (y/n)\n")
resp=prompt()
if (resp == "y") :
graph.delete_all()
elif len(args) > 1 :
n = int(args[1])-1
if n < len(nodes) :
print("Deleting Task('%s')" %(nodes[n]['title']))
graph.delete(nodes[n])
elif args[0] == "dep" :
if len(args) > 1 :
tgt = int(args[1])-1
for dep in args[2:] :
n = int(dep)-1
if n < len(nodes) and tgt < len(nodes):
print("Task('%s') <- Task('%s')" %(nodes[tgt]['title'], nodes[n]['title']))
a_dep_b = Relationship(nodes[tgt], "depends_on", nodes[n])
graph.create(a_dep_b)
elif args[0] == "done" :
if len(args) > 1 :
id = int(args[1])
class User(object):
def __init__(self, graph_db=None):
self.name = ''
self.call_sign = ''
self.first_name = ''
self.last_name = ''
self.id = ''
self.mission_statement = ''
self.about = ''
self.email = ''
self.is_mentor = False
self.is_tutor = False
self.is_visible = True
self.is_available_for_in_person = True
# self._interests_list = None
# self.is_admin = False
# self.password = ''
# self.salt = ''
# self.permanent_web_token = ''
# self.temporary_web_token = ''
self.join_date = None
self.last_active_date = ''
self._graph_db = Graph(settings.DATABASE_URL)
@property
def user_properties(self):
"""
setup user properties
:return: dictionary of properties
"""
properties_dict = dict(self.__dict__)
del properties_dict['_graph_db']
return properties_dict
def set_user_properties(self, user_properties):
"""
:param user_properties:
:return:
"""
for key, value in user_properties.iteritems():
setattr(self, key, value)
def get_user(self):
user_node = self.user_node
if user_node is not None:
user_properties = dict(user_node.properties)
for key, value in user_properties.iteritems():
setattr(self, key, value)
return True
else:
return False
def create_user(self, user_properties=None):
"""
create a new user based on the attributes
:return: node
"""
#TODO exception handling
self.join_date = datetime.date.today()
self.last_active_date = self.join_date
self.id = str(uuid.uuid4())
if user_properties is not None:
self.set_user_properties(user_properties)
new_user_node = Node.cast(GraphLabel.USER, self.user_properties)
try:
self._graph_db.create(new_user_node)
except:
pass
# print 'node probably found. see message'
# print sys.exc_info()
return new_user_node
@property
def user_node(self):
"""
get a user Node
:return: py2neo Node
"""
if self.email != '':
return self._graph_db.find_one(GraphLabel.USER,
property_key='email',
property_value=self.email)
elif self.id != '':
return self._graph_db.find_one(GraphLabel.USER,
property_key='id',
property_value=self.id)
# return self.graph_db.get_or_create_indexed_node(index_name=GraphLabel.USER,
# key='email', value=self.email)
@property
def user_interests(self):
""" get user interests
:return: dictionary of interests
"""
user_interests = self._graph_db.match(start_node=self.user_node,
rel_type=GraphRelationship.INTERESTED_IN,
end_node=None)
#create a list of tuples of interests and the users's relationship to them
interests_list = []
for rel in user_interests:
interest_dict = dict(rel.end_node.properties, **rel.properties)
interests_list.append(dict(rel.end_node.properties))
return interests_list
@property
def user_goals(self):
""" get user interests
:return: list of interests
"""
#TODO do not need a list of interests -- HATEOAS -- MMD 3/8/2015
user_goals = self._graph_db.match(start_node=self.user_node, rel_type=GraphRelationship.HAS_GOAL,
end_node=None)
goals_list = []
goal_interests_list = []
for rel in user_goals:
goal_properties = dict(rel.end_node.properties)
goal = Goal()
goal.id = goal_properties['id']
interests = goal.goal_interests
interests_list = []
for interest in interests:
interests_list.append(interest['name'])
goal_properties['interests'] = interests_list
goals_list.append(goal_properties)
return goals_list
@property
def user_groups(self):
"""
:return: list of tuples of the groups
"""
#TODO add list of related interests
user_groups = self._graph_db.match(start_node=self.user_node, rel_type=GraphRelationship.STUDIES_WITH,
end_node=None)
# create a list of tuples of interests and the users's relationship to them
groups_list = []
for rel in user_groups:
group_properties = dict(rel.end_node.properties)
group = Group()
group.id = group_properties['id']
interests = group.group_interests
group_interests_list = []
for interest in interests:
group_interests_list.append(interest['name'])
group_properties['interests'] = group_interests_list
groups_list.append(group_properties)
return groups_list
@property
def user_orgs(self):
"""
:return:
"""
user_orgs = self._graph_db.match(start_node=self.user_node,
rel_type=GraphRelationship.MEMBER_OF,
end_node=None)
orgs_list = []
for rel in user_orgs:
org_properties = dict(rel.end_node.properties)
org = Organization()
org.id = org_properties['id']
interests = org.org_interests
interests_list = []
for interest in interests:
interests_list.append(interest['name'])
org_properties['interests'] = interests_list
orgs_list.append(org_properties)
return orgs_list
@property
def user_locations(self):
"""
:return:
"""
user_locations = self._graph_db.match(start_node=self.user_node,
rel_type=GraphRelationship.LOCATED_IN,
end_node=None)
locations_list = []
for rel in user_locations:
locations_list.append(rel.end_node.properties)
return locations_list
def get_local_users_shared_interests_near_location(self): #, location_node):
"""
get a dictionary of user with shared interests with this user
:param :
:return: dictionary of {interests: [users]}
"""
# users_shared_interests
params = {
'email': '*****@*****.**'
}
cypher_str = "MATCH (u:USER {email:{email}})-[url:LOCATED_IN]->(l:LOCATION)"
cypher_str += "<-[orl:LOCATED_IN]-(o:USER) "
cypher_str += "WITH u, o, l, url, orl "
cypher_str += "MATCH (u)-[ru:INTERESTED_IN]->"
cypher_str += "(i:INTEREST)<-[ro:INTERESTED_IN]-(o) "
cypher_str += "RETURN i.name as interest_name, i.id as interest_id, " \
"o.name as user_name, o.id as user_id" #, u, ru, ro, l, url, orl"
# print cypher_str
results = Graph().cypher.execute(cypher_str, params)
# self.graph_db.cypher.stream(cypher_str)
# self.graph_db.cypher.execute(cypher_str)
interest_users_dict = {}
print results
for item in results:
interest = item['interest_name']
user = item['user_name']
if interest_users_dict.has_key(interest):
interest_users_dict[interest].append(user)
else:
interest_users_dict[interest] = []
interest_users_dict[interest].append(user)
# user = item['user_name']
# cur_users_list.append(interest_users_dict.get(interest))
# if interest_users_dict.has_key(interest):
# # if interest in interest_users_dict.keys():
# cur_users_list = interest_users_dict[interest]
# # cur_users_list = interest_users_dict.get(interest)
# else:
# interest_users_dict[interest] = []
# cur_users_list.append(user)
# interest_users_dict[interest] = cur_users_list
# interest_users_dict[interest] = interest_users_dict.get(interest)
# user_details = (user_node['name'], user_node['email'], user_node['id'])
# user_list.append(user_details)
return interest_users_dict
def add_interest(self, interest_id, experience_properties_dict=None):
""" Add interest to user
:param interest id:string uuid
:return: List of interests
"""
#TODO add exception handling
interest = Interest()
interest.id = interest_id
interest_node = interest.interest_node_by_id
user_interest_relationship = Relationship(self.user_node,
GraphRelationship.INTERESTED_IN,
interest_node)
for key, value in experience_properties_dict.iteritems():
user_interest_relationship[key] = value
try:
self._graph_db.create_unique(user_interest_relationship)
except:
pass
return self.user_interests
def update_interest(self, interest_id, experience_properties_dict):
interest = Interest()
interest.id = interest_id
interest_node = interest.interest_node_by_id
user_interest_relationship = self._graph_db.match_one(start_node=self.user_node,
rel_type=GraphRelationship.INTERESTED_IN,
end_node=interest_node)
for key, value in experience_properties_dict.iteritems():
user_interest_relationship.properties[key] = value
user_interest_relationship.push()
def delete_interest(self, interest_id):
"""
drop interest relationship from user given the interest_id
:param interest_id: str(uuid.uuid4())
:return:
"""
#TODO exception handling
interest = Interest()
interest.id = interest_id
interest_node = interest.interest_node_by_id
user_interest_relationship = self._graph_db.match_one(start_node=self.user_node,
rel_type=GraphRelationship.INTERESTED_IN,
end_node=interest_node)
self._graph_db.delete(user_interest_relationship)
def update_user(self):
user_node = self.user_node
user_properties = dict(self.user_properties)
for key, value in user_properties.iteritems():
user_node[key] = value # user_properties[key]
user_node.push()
# def make_admin(self):
# #new_user = self.graph_db.get_or_create_indexed_node(index_name=GraphLabel.USER, key='email', value=self.email)
# self.user_node.add_labels(GraphLabel.ADMIN)
def add_goal(self, goal_properties):
"""
Add goal to user
:param goal_id: string uuid
:return: List of user goals
"""
#TODO exception handling
goal = Goal()
goal.set_goal_properties(goal_properties=goal_properties)
goal.create_goal()
# create relationship between user and interest node
user_goal_relationship = Relationship(self.user_node,
GraphRelationship.HAS_GOAL,
goal.goal_node)
self._graph_db.create_unique(user_goal_relationship)
#TODO set properties on the relationship -- may use a unique id as the key
return self.user_goals
def delete_goal(self, goal_id):
user_node = self.user_node
goal = Goal()
goal.id = goal_id
# have to remove all relationships before deleteing a node
goal.delete_all_interests()
goal_node = goal.goal_node
user_goal_rel = self._graph_db.match_one(start_node=user_node,
rel_type=GraphRelationship.HAS_GOAL,
end_node=goal_node)
self._graph_db.delete(user_goal_rel)
self._graph_db.delete(goal_node)
def join_group(self, group_id, group_relationship_properties=None):
"""
Add user as member of group
:param group_id: string uuid
:return:
"""
#TODO exception handling
group = Group()
group.id = group_id
# relationship properties
join_properties = {
'join_date': datetime.date.today()
}
user_group_relationship = Relationship(self.user_node,
GraphRelationship.STUDIES_WITH,
group.group_node)
# properties=join_properties)
for key, value in join_properties.iteritems():
user_group_relationship[key] = value
try:
self._graph_db.create_unique(user_group_relationship)
except:
pass
#TODO set properties on the relationsip
# group_relationship_properties["id"] = str(uuid.uuid4())
def leave_group(self, group_id):
"""
remove relationship between user and study group
:param group_id: string uuid
:return: None
"""
#TODO exception handling
group = Group()
group.id = group_id
user_group_relationship = self._graph_db.match_one(start_node=self.user_node,
rel_type=GraphRelationship.MEMBER_OF,
end_node=group.group_node)
self._graph_db.delete(user_group_relationship)
def delete_group(self, group_id):
pass
def join_organization(self, organization_id):
"""
add user to organization
:param organization_id: string uuid
:return: list of tuple of interests
"""
#TODO exception handling
org = Organization()
org.id = organization_id
user_org_relationship = Relationship(self.user_node,
GraphRelationship.MEMBER_OF,
org.org_node)
try:
self._graph_db.create_unique(user_org_relationship)
except:
print sys.exc_info()[0]
def leave_organization(self, organization_id):
"""
remove relationship between user and organization
:param organization_id:
:return:
"""
#TODO exception handling
org = Organization()
org.id = organization_id
user_org_relationship = self._graph_db.match_one(start_node=self.user_node,
rel_type=GraphRelationship.MEMBER_OF,
end_node=org.org_node)
self._graph_db.delete(user_org_relationship)
def add_location(self, location_json):
"""
link user to location nodes
:param locations_place_id:
:return:
"""
#TODO exception handling
#TODO do in location and pass in the node from the actual object (better pattern)
location_place_id = location_json['id']
location = Location()
location.id = location_place_id
location_node = location.location_node_by_place_id
if not location_node:
location.set_location_properties(location_json)
location.create_location()
location_node = location.location_node_by_place_id()
user_location_relationship = Relationship(self.user_node,
GraphRelationship.LOCATED_IN,
location_node)
# try:
self._graph_db.create_unique(user_location_relationship)
# except:
# pass
def create_cq(self, cq_dict, cq_interests_dict=None):
Cq.create_cq(user_node=self.user_node, cq_dict=cq_dict)
def create_converation_between_users(self, user_id_started, user_id_with, conversation_properties):
# self.id = uuid.uuid4()
conversation_properties['id'] = str(uuid.uuid4())
new_convo_node = Node.cast(GraphLabel.CONVERSATION, conversation_properties)
try:
convo_node, = self._graph_db.create(new_convo_node) # create new conversation node
user_started = User()
user_started.id = user_id_started
user_with = User()
user_with.id = user_id_with
# create started conversation relationship
user_started_relationship = Relationship(user_started.user_node,
GraphRelationship.STARTED,
convo_node)
self._graph_db.create(user_started_relationship)
# create started conversation with relationship
convo_with_relationship = Relationship(convo_node,
GraphRelationship.WITH,
user_with.user_node)
self._graph_db.create(convo_with_relationship)
return convo_node.properties['id']
except:
pass #TODO add exception handling
# @staticmethod
def matched_users(self, match_string, limit):
"""
:param match_string:
:param limit:
:return: dictionary of search results
"""
params = {
'match': '(?i)%s.*' % match_string,
'limit': limit
}
cypher_str = "MATCH (user:USER ) " \
"WHERE user.name =~ {match} " \
"RETURN user.name as name, user.id as id " \
"LIMIT {limit}"
match_results = self._graph_db.cypher.execute(statement=cypher_str, parameters=params)
root = {}
root['count'] = 0
user_found = {}
users_list = []
for item in match_results:
user_found['id'] = item.id
user_found['name'] = item.name
# self.id = item['id']
# self.get_user()
# users_list.append(dict(self.user_properties))
users_list.append(dict(user_found))
root['count'] += 1
root['users'] = users_list
return root
def register_user(self, email):
verification_email = notifications.Notifications()
verification_email.recipients = [email]
s = URLSafeTimedSerializer(secret_key=settings.TOKEN_SECRET_KEY)
payload = s.dumps(email)
verification_email.subject = settings.ACTIVATION_SUBJECT
verification_email.message = settings.ACTIVATION_MESSAGE
verification_email.url = self.construct_verification_url(payload=payload)
verification_email.send_by_gmail()
def activate_user(self, payload, email):
s = URLSafeTimedSerializer(secret_key=settings.TOKEN_SECRET_KEY)
payload_email = s.loads(payload, max_age=settings.TOKEN_EXPIRES_IN) # 10 minutes
if email == payload_email:
self.email = email
self.get_user()
self.permanent_web_token = self.create_web_token()
if self.id == '':
self.create_user()
else:
self.update_user()
else:
raise BadSignature('bad email')
def update_last_active_date(self):
self.last_active_date = datetime.date.today()
user_node = self.user_node
user_node['last_active_date'] = self.last_active_date
user_node.push()
def construct_verification_url(self, payload):
return settings.SITE_URL + settings.ACTIVATION_ROUTE + "/%s" % payload
def create_web_token(self):
s = URLSafeSerializer(secret_key=settings.TOKEN_SECRET_KEY)
return s.dumps(self.id)
def user_relationships_for_json(self, auth_id):
root = self.user_profile_for_json()
root['__class'] = self.__class__.__name__
root['interests'] = self.user_interests
root['locations'] = self.user_locations
root['goals'] = self.user_goals
root['groups'] = self.user_groups
root['organizations'] = self.user_orgs
root['is_owner'] = (auth_id == self.id)
root['allow_edit'] = (auth_id == self.id)
root['allow_message'] = (auth_id is not None)
return root
def user_profile_for_json(self):
root = self.user_properties
return root
def user_interests_for_json(self):
root = {}
root['__class'] = self.__class__.__name__
root['id'] = self.id
root['email'] = self.email
root['interests'] = self.user_interests
return root
def user_goals_for_json(self):
root = {}
root['__class'] = self.__class__.__name__
root['id'] = self.id
root['email'] = self.email
root['goals'] = self.user_goals
# root['interests'] = self.user_goals['interests']
return root
def user_groups_for_json(self):
root = {}
root['__class'] = self.__class__.__name__
root['id'] = self.id
root['email'] = self.email
root['groups'] = self.user_groups
return root
def user_locations_for_json(self, auth_id):
root = {}
root['__class'] = self.__class__.__name__
root['id'] = self.id
if self.id == auth_id:
root ['allow_edit'] = True
return root
def local_users_with_shared_interests_for_json(self):
root = {}
root['__class'] = self.__class__.__name__
root['id'] = self.id
root['email'] = self.email
root['users'] = self.get_local_users_shared_interests_near_location()
return root
def activated_user_for_json(self):
root = {}
root['__class'] = self.__class__.__name__
root['x_auth_key'] = self.permanent_web_token
return root
class GraphdbClient(object):
@LazyProperty
def __db_host__(self):
return self.db_host
@LazyProperty
def __db_port__(self):
return self.db_port
@LazyProperty
def __db_user__(self):
return self.db_user
@LazyProperty
def __db_password__(self):
return self.db_password
def get_db(self):
return self.graph
def __init__(self,
db_host='localhost',
db_port=27017,
db_user='',
db_password=''):
self._setDatabase(db_host, db_port, db_user, db_password)
def _setDatabase(self, db_host, db_port, db_user, db_password):
self.db_host = db_host
self.db_port = db_port
self.db_user = db_user
self.db_password = db_password
self.graph = Graph(host=self.db_host, bolt=True, http=self.db_port, \
user=self.db_user, password=self.db_password)
self.log = LogHandler("graphdb_client", level=20)
def insert_or_update_node(self, label, key, vals={}):
"""
Insert if the node is not in the dataset or update the existed relationship.
Parameters:
label: label;
key: key;
properties: json string.
Example: insert_or_update_node("Test", "yexuliang", {"Test":"123123", "has_crawled" : 1})
"""
nd = Node(label, neo_id="{}_{}".format(label.lower(), key))
self.graph.merge(nd)
for k, v in vals.iteritems():
val = list()
if isinstance(v, list):
if isinstance(v[0], dict):
for elem in v:
val.append(json.dumps(elem))
else:
val = v
nd[k] = val
self.graph.push(nd)
return nd
def insert_or_update_relation(self, rel_type, st_nd, end_nd, vals={}):
"""
Insert if the relationship is not in the dataset or update the existed relationship.
Parameters:
rel_type: type of relationship;
st_nd: start node;
end_nd: end node;
bidirectional: True or False;
vals: json string.
Example: insert_or_update_relation("T_Rel", Node("Test", uid="yexuliang"), Node("Test", uid="3310858"), {"clue":["id", "screen_name"]})
"""
rel = Relationship(st_nd, rel_type, end_nd)
self.graph.merge(rel)
for k, v in vals.iteritems():
rel[k] = v
self.graph.push(rel)
def find_node_by_id(self, label, key):
"""
Find nodes by neo_id.
Parameters:
label: label;
key: neo_id.
Example: find_node_by_id("Douban", "Weibo_3513921")
"""
nds = self.graph.data(
"MATCH (nd:{} {{neo_id: '{}'}}) RETURN nd".format(label, key))
if nds:
return nds[0]['nd']
else:
return None
def find_node_by_rel(self,
nd_label,
nd_info,
rel_type,
rel_info={},
is_count=False,
limit=None,
skip=None):
"""
Find node by conditions.
Parameters:
label: label;
nd_info: dictionary where key is the property_name and value is the property_val;
rel_info: the filter of aligned info
is_count: if False, return nodes; else return num of queried results
limit: default None. limited number of query results.
skip: for return results by segment
Example: find_node_by_rel("Douban", {"name":"=~'ta'"}, 'ALIGN', {'ID': '>.9'})
"""
cql = "MATCH (nd:{})-[a:{}]-(res)".format(nd_label, rel_type)
nd_condition_clause = condition_clause_format('nd', nd_info)
rel_cond_clause = condition_clause_format('a', rel_info)
condition_clause = nd_condition_clause
if rel_cond_clause:
condition_clause += " AND " + rel_cond_clause
if is_count:
ret_clause = " RETURN count(nd) as num"
else:
ret_clause = " RETURN res"
if condition_clause:
cql += " WHERE " + condition_clause + ret_clause
else:
cql += ret_clause
if skip and isinstance(skip, int):
cql += " SKIP {}".format(skip)
if limit and isinstance(limit, int):
cql += " LIMIT {}".format(limit)
self.log.debug(u"Query: %s" % cql)
nds = self.graph.run(cql)
if is_count:
return nds
graph_nds = list()
for nd in nds:
graph_nds.append(nd['res'])
if graph_nds:
return graph_nds
else:
return None
def find_node_by_property(self,
label,
nd_info=dict(),
is_count=False,
limit=None,
skip=None):
"""
Find node by conditions.
Parameters:
label: label;
nd_info: dictionary where key is the property_name and value is the property_val;
is_count: if False, return nodes; else return num of queried results
limit: default None. limited number of query results.
skip: for return results by segment
Example: find_node_by_property("Douban", {"name":"=~'ta'"}, False)
"""
cql = "MATCH (nd:{})".format(label)
condition_clause = condition_clause_format('nd', nd_info)
if is_count:
ret_clause = " RETURN count(nd) as num"
else:
ret_clause = " RETURN nd"
if condition_clause:
cql += " WHERE " + condition_clause + ret_clause
else:
cql += ret_clause
if skip and isinstance(skip, int):
cql += " SKIP {}".format(skip)
if limit and isinstance(limit, int):
cql += " LIMIT {}".format(limit)
self.log.debug(u"Query: %s" % cql)
nds = self.graph.data(cql)
if is_count:
return nds
graph_nds = list()
for nd in nds:
graph_nds.append(nd['nd'])
if graph_nds:
return graph_nds
else:
return None
# @deprecated(find_node_by_property)
# def find_node_by_fuzzy_property(self, label, nd_info, limit=None):
# """
# Find node by fuzzy conditions.
# Parameters:
# label: label;
# nd_info: dictionary where key is the property_name and value is the property_val;
# limit: default None. limited number of query results.
# Example: find_node_by_fuzzy_property("Douban", {"name":"ta"}, 1)
# """
# cql = "MATCH (nd:{})".format(label)
# condition_clause = fuzzy_condition_clause_format('nd', nd_info)
# if condition_clause:
# cql += " WHERE "+condition_clause+" RETURN nd"
# else:
# cql += " RETURN nd"
# if limit:
# cql += " LIMIT {}".format(limit)
# self.log.info(u"Query: %s"%cql)
# nds = self.graph.data(cql)
# if nds:
# return nds
# else:
# return None
# @deprecated(find_node_by_property)
# def find_node_by_acc_property(self, label, nd_info, limit=None):
# """
# Find node by accurate conditions.
# Parameters:
# label: label;
# nd_info: dictionary where key is the property_name and value is the property_val;
# limit: default None. limited number of query results.
# Example: find_rel_by_acc_property('id', {'label':'Douban', 'name':'tada'},\
# {'label':'Weibo', 'nick_name':'tadamiracle'})
# """
# cql = "MATCH (nd:{})".format(label)
# condition_clause = acc_condition_clause_format('nd', nd_info)
# if condition_clause:
# cql += " WHERE "+condition_clause+" RETURN nd"
# else:
# cql += " RETURN nd"
# if limit:
# cql += " LIMIT {}".format(limit)
# self.log.info(u"Query: %s"%cql)
# nds = self.graph.data(cql)
# if nds:
# return nds
# else:
# return None
def find_rel_by_property(self,
st_nd,
end_nd,
rel_type='ALIGN',
rel_info=dict(),
is_count=False,
limit=None,
skip=None):
"""
Find relationship by conditions.
Parameters:
st_nd: start node info in dictionary where key is the property_name and value is the property_val.
end_nd: end node info in dictionary where key is the property_name and value is the property_val.
rel_type: the type of relationship
rel_info: the filter of relationship info
is_count: if False, return relationships; else return count(rel)
limit: default None. limited number of query results
skip: for return results by segment
Example: find_rel_by_property({'label':'Douban', 'name':"=~'.*ta.*'"}, {'label':'Weibo'})
"""
if 'label' in st_nd and 'label' in end_nd:
cql = "MATCH (st_nd:{})-[rel:{}]-(end_nd:{})".format(
st_nd['label'], rel_type, end_nd['label'])
else:
self.log.warning(
u"No specific relation type in 'find_rel_by_property'")
return None
st_cond_clause = condition_clause_format('st_nd', st_nd)
end_cond_clause = condition_clause_format('end_nd', end_nd)
rel_cond_clause = condition_clause_format('rel', rel_info)
condition_clause = st_cond_clause
if end_cond_clause:
condition_clause += " AND " + end_cond_clause
if rel_info:
condition_clause += " AND " + rel_info
if is_count:
ret_clause = " RETURN count(rel) as num"
else:
ret_clause = " RETURN st_nd, rel, end_nd"
if condition_clause:
cql += " WHERE " + condition_clause + ret_clause
else:
cql += ret_clause
if skip and isinstance(skip, int):
cql += " SKIP {}".format(skip)
if limit and isinstance(limit, int):
cql += " LIMIT {}".format(limit)
self.log.debug(u"Query: %s" % cql)
rels = self.graph.data(cql)
graph_rels = list()
for rel in rels:
graph_rels.append(rel['rel'])
if graph_rels:
return graph_rels
else:
return None
# @deprecated(find_rel_by_property)
# def find_rel_by_fuzzy_property(self, rel_type, st_nd, end_nd, rel_info, limit=None):
# """
# Find relationship by fuzzy conditions.
# Parameters:
# rel_type: relation type;
# st_nd: start node info in dictionary where key is the property_name and value is the property_val.
# end_nd: end node info in dictionary where key is the property_name and value is the property_val.
# limit: default None. limited number of query results.
# Example: find_rel_by_fuzzy_property('id', {'label':'Douban', 'name':'ta'}, {'label':'Weibo'}, 2)
# """
# if 'label' in st_nd and 'label' in end_nd:
# cql = "MATCH (st_nd:{})-[a:ALIGN]-(end_nd:{})".format(st_nd['label'], end_nd['label'])
# else:
# self.log.warning(u"No specific relation type in 'find_rel_by_fuzzy_property'")
# return None
# st_cond_clause = fuzzy_condition_clause_format('st_nd', st_nd)
# end_cond_clause = fuzzy_condition_clause_format('end_nd', end_nd)
# condition_clause = st_cond_clause
# if end_cond_clause:
# condition_clause += " AND "+end_cond_clause
# if rel_type:
# condition_clause += " AND '{}' in a.align_msg".format(rel_type)
# if condition_clause:
# cql += " WHERE "+condition_clause+" RETURN st_nd, a, end_nd"
# else:
# cql += " RETURN st_nd, a, end_nd"
# if limit:
# cql += " LIMIT {}".format(limit)
# self.log.info(u"Query: %s"%cql)
# rels = self.graph.data(cql)
# if rels:
# return rels
# else:
# return None
# @deprecated(find_rel_by_property)
# def find_rel_by_acc_property(self, rel_type, st_nd, end_nd, limit=None):
# """
# Find relationship by accurate conditions.
# Parameters:
# rel_type: relation type;
# st_nd: start node info in dictionary where key is the property_name and value is the property_val.
# end_nd: end node info in dictionary where key is the property_name and value is the property_val.
# limit: default None. limited number of query results.
# Example: find_rel_by_fuzzy_property('id', {'label':'Douban', 'name':'ta'}, {'label':'Weibo'}, 2)
# """
# if 'label' in st_nd and 'label' in end_nd:
# cql = "MATCH (st_nd:{})-[a:ALIGN]->(end_nd:{})".format(st_nd['label'], end_nd['label'])
# else:
# self.log.warning(u"No specific relation type in 'find_rel_by_acc_property'")
# return None
# st_cond_clause = acc_condition_clause_format('st_nd', st_nd)
# end_cond_clause = acc_condition_clause_format('end_nd', end_nd)
# condition_clause = st_cond_clause
# if end_cond_clause:
# condition_clause += " AND "+end_cond_clause
# if rel_type:
# condition_clause += " AND '{}' in a.align_msg".format(rel_type)
# cql += " WHERE "+condition_clause+" RETURN st_nd, a, end_nd"
# if limit:
# cql += " LIMIT {}".format(limit)
# self.log.info(u"Query: %s"%cql)
# rels = self.graph.data(cql)
# if rels:
# return rels
# else:
# return None
def delete(self, subgraph):
if isinstance(subgraph, Relationship):
self.graph.separate(subgraph)
else:
self.graph.delete(subgraph)
def clear(self):
self.graph.delete_all()
# has_contrast = models.Relationship(Contrast,rel_type='HASCONTRAST')
for row in conditions.iterrows():
name = row[1].condition_text
user = row[1].id_user
uid = row[1].id
task = row[1].id_term
description = row[1].condition_description
timestamp = row[1].event_stamp
if not str(name) =="nan":
properties = {"description":description}
node = make_node("condition",uid,name,properties)
tasknode = find_node("task",property_value=task)
# If the tasknode is node found, then we do not create the relation, and we delete the condition
if tasknode == None:
graph.delete(node)
else:
relation = make_relation(tasknode,"HASCONDITION",node)
#class Disorder(models.NodeModel):
# has_difference = models.Relationship(Contrast,rel_type='HASDIFFERENCE')
# mentioned_in = models.Relationship('PMID',rel_type='MENTIONEDIN')
disorders.columns = ["id","term","classification"] # Bug in returning column index name
for row in disorders.iterrows():
uid = row[1].id
classification = row[1].classification
name = row[1].term
properties = {"classification":classification}
node = make_node("disorder",uid,name,properties)
class HW3Graph(object):
"""
This object provides a set of helper methods for creating and retrieving nodes and relationships from
a Neo4j database holding information about players, teams, fans, comments and their relationships.
"""
# Note:
# I tend to avoid object mapping frameworks. Object mapping frameworks are fun in the beginning
# but tend to be annoying after a while. So, I did not create types Player, Team, etc.
#
# Connects to the DB and sets a Graph instance variable.
# Also creates a NodeMatcher and RelationshipMatcher, which are a py2neo framework classes.
def __init__(self, auth=('neo4j', 'dbuserdbuser'), host='localhost', port=11005, secure=False, ):
self._graph = Graph(secure=secure,
bolt=True,
auth=auth,
host=host,
port=port)
self._node_matcher = NodeMatcher(self._graph)
self._relationship_matcher = RelationshipMatcher(self._graph)
def run_q(self, qs, args):
"""
:param qs: Query string that may have {} slots for parameters.
:param args: Dictionary of parameters to insert into query string.
:return: Result of the query, which executes as a single, standalone transaction.
"""
try:
tx = self._graph.begin(autocommit=False)
result = self._graph.run(qs, args)
return result
except Exception as e:
print("Run exaception = ", e)
def run_match(self, labels=None, properties=None):
"""
Uses a NodeMatcher to find a node matching a "template."
:param labels: A list of labels that the node must have.
:param properties: A dictionary of {property_name: property_value} defining the template that the
node must match.
:return: An array of Node objects matching the pattern.
"""
# ut.debug_message("Labels = ", labels)
# ut.debug_message("Properties = ", json.dumps(properties))
if labels is not None and properties is not None:
result = self._node_matcher.match(labels, **properties)
elif labels is not None and properties is None:
result = self._node_matcher.match(labels)
elif labels is None and properties is not None:
result = self._node_matcher.match(**properties)
else:
raise ValueError("Invalid request. Labels and properties cannot both be None.")
# Convert NodeMatch data into a simple list of Nodes.
full_result = []
for r in result:
full_result.append(r)
return full_result
def find_nodes_by_template(self, tmp):
"""
:param tmp: A template defining the label and properties for Nodes to return. An
example is { "label": "Fan", "template" { "last_name": "Ferguson", "first_name": "Donald" }}
:return: A list of Nodes matching the template.
"""
labels = tmp.get('label', None)
props = tmp.get("template", None)
result = self.run_match(labels=labels, properties=props)
return result
def create_node(self, label, **kwargs) -> object:
n = Node(label, **kwargs)
tx = self._graph.begin(autocommit=True)
tx.create(n)
return n
# Created method by Tim
def create_relationship(self, a, r, b) -> object:
g = HW3Graph()
tx = g._graph.begin(autocommit=True)
ab = Relationship(a, r, b)
# ab is subgraph
tx.create(ab)
# Created method by Tim
def find_by_relationship(self, template, r) -> object:
key, value = list(template['template'].items())[0]
tx = self._graph.begin(autocommit=True)
qs = 'MATCH (n:' + template['label'] + ' {' + key + ':"' + value + '"})-[:' + r + \
']->(node) RETURN node LIMIT 100'
res = tx.run(cypher=qs)
return res.data()
# Created method by Tim
def delete_node(self, tmp):
"""
:param tmp: A template defining the label and properties for Nodes to return. An
example is { "label": "Fan", "template" { "last_name": "Ferguson", "first_name": "Donald" }}
:return: A list of Nodes matching the template.
"""
flag = False
subgraph = self.find_nodes_by_template(tmp)
if subgraph:
self._graph.delete(subgraph)
flag = True
return flag
def update(self, old_tmp, new_tmp):
"""
:param tmp: A template defining the label and properties for Nodes to return. An
example is { "label": "Fan", "template" { "last_name": "Ferguson", "first_name": "Donald" }}
:return: A list of Nodes matching the template.
"""
flag = self.delete_node(old_tmp)
result = None
if flag:
result = self.create_node(new_tmp)
return result
class GetData(object):
def __init__(self, uri, user, pwd, curTxBlockNum):
self.graph = Graph(uri, username=user, password=pwd)
self.matcher_node = NodeMatcher(self.graph)
self.matcher_relation = RelationshipMatcher(self.graph)
self.curTxBlockNum = curTxBlockNum
# self.graph.delete_all()
self.jsCode = '''
var hashjs = require('hash.js');
function getAddress(pubKey) {
return hashjs.sha256().update(pubKey, 'hex').digest('hex').slice(24);
}
'''
self.eval = execjs.compile(self.jsCode)
# 输出至控制台以及log 文件中
def printf(self, message):
# print(message)
with open("../log/%s.log" % time.strftime('%Y-%m-%d',time.localtime(time.time())), "a") as file:
file.write(message + '\r')
def getAddress(self, address, type_):
if type_ == "pubKey":
old_address = self.eval.call("getAddress", address[2:])
account = Account(address=old_address)
return (old_address, account.bech32_address)
elif type_ == "toBech32":
if is_bech32_address(address): # no need to transfer to bech32 format
return (address, address)
account = Account(address=address)
return (address, account.bech32_address)
def getResult(self, method, params):
data["method"] = method
data["params"] = [params]
try:
res = requests.post("https://api.zilliqa.com/", json=data, timeout=10).json()
return res["result"] if "result" in res else res
except requests.exceptions.RequestException as e:
self.printf("ERROR: Api Connect Timeout: Recall Api!")
return self.getResult(method, params)
def getTxBlcokData(self, startBlock, endBlock):
for i in range(startBlock, endBlock+1):
self.getOneTxBlcokData(str(i))
def getOneTxBlcokData(self, blockNum):
self.printf("the %s TxBlock %s" % (blockNum, time.strftime("%y-%m-%d %H:%M:%S", time.localtime())))
# 若该Block存在，则删除
txBlock = self.matcher_node.match("TxBlock", BlockNum=blockNum).first()
if txBlock:
self.printf("TxBlock \t%s existed!" % blockNum)
self.graph.delete(txBlock)
res = self.getResult("GetTxBlock", blockNum)
while "header" not in res or res["header"]["Timestamp"] == "0":
# 表示到达最新的一个块，等一分钟再继续更新
self.printf("Waiting for Next Block %s" % time.strftime("%y-%m-%d %H:%M:%S", time.localtime()))
time.sleep(60)
res = self.getResult("GetTxBlock", blockNum)
# 创建TxBlock节点以及与所属DSBlock之间的从属关系
TxBlock = Node("TxBlock")
TxBlock.update({"HeaderSign": res["body"]["HeaderSign"]})
TxBlock.update(res["header"])
# 由于一个TxBlock块由多个MicroBlock组成
for index, block in enumerate(res["body"]["MicroBlockInfos"]):
TxBlock.update({"MicroBlock%dHash" % index: block["MicroBlockHash"], "MicroBlock%dTxnRootHash" % index: block["MicroBlockTxnRootHash"]})
DsBlock = self.matcher_node.match("DsBlock", BlockNum=res["header"]["DSBlockNum"]).first()
# 更新当前TxBlock 所对应的DsBlock
if not DsBlock:
DsBlock = self.getOneDsBlcokData(res["header"]["DSBlockNum"])
Tx2Ds = Relationship(TxBlock, "in", DsBlock)
Ds2Tx = Relationship(DsBlock, "has", TxBlock)
##########################################################
# 为has 和in 属性添加排序信息，即TxBlock 是DSBlock 中的第几个
# cur_number = len(self.matcher_relation.match([DsBlock], "has"))
order = int(blockNum) % 100
Tx2Ds["order"] = order+1
Ds2Tx["order"] = order+1
# 创建Miner节点及其与TxBlock节点之间的挖与被挖关系
miner_addr_old, miner_addr_new = self.getAddress(res["header"]["MinerPubKey"], type_="pubKey")
Miner, existed = self.getAccountData(miner_addr_old, miner_addr_new)
Miner2Tx = Relationship(Miner, "mine", TxBlock)
Tx2Miner = Relationship(TxBlock, "mined", Miner)
self.graph.create(TxBlock)
self.graph.create(Tx2Ds)
self.graph.create(Ds2Tx)
if not existed:
self.graph.create(Miner)
self.graph.create(Miner2Tx)
self.graph.create(Tx2Miner)
# 当TxBlock中有交易时，获取交易信息以及交易双方的账户信息并存入Neo4j中
if res["header"]["NumTxns"]:
self.printf("%d Txs in all" % res["header"]["NumTxns"])
self.getTxData(res["header"]["BlockNum"])
def getDsBlockData(self, startBlock, endBlock):
for i in range(startBlock, endBlock+1):
self.getOneDsBlcokData(str(i))
# 返回DsBlock 节点
def getOneDsBlcokData(self, blockNum):
self.printf("the %s DsBlock %s" % (blockNum, time.strftime("%y-%m-%d %H:%M:%S", time.localtime())))
res = self.getResult("GetDsBlock", blockNum)
# 创建DSBlock节点
DsBlock = Node("DsBlock")
DsBlock.update(res["header"])
DsBlock.update({"signature": res["signature"]})
# 创建Miner节点及其与DsBlock节点之间的挖与被挖关系
miner_addr_old, miner_addr_new = self.getAddress(res["header"]["LeaderPubKey"], type_="pubKey")
Miner, existed = self.getAccountData(miner_addr_old, miner_addr_new)
Miner2Ds = Relationship(Miner, "mine", DsBlock)
Ds2Miner = Relationship(DsBlock, "mined", Miner)
self.graph.create(DsBlock)
if not existed:
self.graph.create(Miner)
self.graph.create(Miner2Ds)
self.graph.create(Ds2Miner)
return self.matcher_node.match("DsBlock", BlockNum=blockNum).first()
def getTxData(self, txBlockNum):
res, order = self.getResult("GetTransactionsForTxBlock", txBlockNum), 1
while "error" in res:
self.printf("ERROR: get microblock failed! recatch txs!" + res["error"]["message"])
res = self.getResult("GetTransactionsForTxBlock", txBlockNum)
for MicroBlock in res:
if not MicroBlock:
continue
for txHash in MicroBlock:
self.getOneTxData(txBlockNum, txHash, order)
order += 1
# 获取一条交易的信息
@func_set_timeout(60)
def getOneTxData(self, txBlockNum, txHash, order):
self.printf("The %s transaction %s" % (txHash, time.strftime("%y-%m-%d %H:%M:%S", time.localtime())))
Tx = self.matcher_node.match("Tx", ID=txHash).first()
if Tx:
self.printf("Tx %s existed!" % txHash)
# Tx 已存在时，仍需建立TxBlock 与Tx 之间的关系
TxBlock = self.matcher_node.match("TxBlock", BlockNum=txBlockNum).first()
Tx2TxBlock = Relationship(Tx, "in", TxBlock)
TxBlock2Tx = Relationship(TxBlock, "has", Tx)
##########################################################
# 为has 和in 属性添加排序信息，即Tx 是TxBlock 中的第几个
Tx2TxBlock["order"] = order
TxBlock2Tx["order"] = order
self.graph.create(Tx2TxBlock)
self.graph.create(TxBlock2Tx)
return
res_ = self.getResult("GetTransaction", txHash)
# 获取交易信息时出错
############################################# 待修改
if "error" in res_:
self.printf("ERROR: get txn info failed!" + res_["error"]["message"])
return -1
# 解决调用api 时可能产生的问题
while "receipt" not in res_:
res_ = self.getResult("GetTransaction", txHash)
# receipt不符合格式 重新规整一下
res_["cumulative_gas"] = res_["receipt"]["cumulative_gas"]
res_["epoch_num"] = res_["receipt"]["epoch_num"]
res_["success"] = str(res_["receipt"]["success"])
res_.pop("receipt")
from_addr_old, from_addr_new = self.getAddress(res_["senderPubKey"], type_="pubKey")
to_addr_old, to_addr_new = self.getAddress(res_["toAddr"], type_="toBech32")
# 创建Tx的节点以及与所属TxBlock之间的从属关系
Tx = Node("Tx")
Tx.update(res_)
# 由于res_["senderPubKey"]不是直接的地址，需要转换一下再将发送者的地址存储，新增bech32 格式地址存储
Tx.update({"fromAddrNew": from_addr_new, "toAddrNew": to_addr_new})
TxBlock = self.matcher_node.match("TxBlock", BlockNum=txBlockNum).first()
Tx2TxBlock = Relationship(Tx, "in", TxBlock)
TxBlock2Tx = Relationship(TxBlock, "has", Tx)
##########################################################
# 为has 和in 属性添加排序信息，即Tx 是TxBlock 中的第几个
Tx2TxBlock["order"] = order
TxBlock2Tx["order"] = order
# 创建账户节点以及所参与Tx之间的发送接收关系，其中返回中
# 有一个标记信息，表示数据库中是否已经存在该节点
From_Account, existed_from = self.getAccountData(from_addr_old, from_addr_new)
To_Account, existed_to = self.getAccountData(to_addr_old, to_addr_new)
From2Tx = Relationship(From_Account, "send", Tx)
Tx2From = Relationship(Tx, "from", From_Account)
To2Tx = Relationship(To_Account, "receive", Tx)
Tx2To = Relationship(Tx, "to", To_Account)
From2To = Relationship(From_Account, "traded", To_Account)
###################################################
# 之前便加过TxHash
From2To['TxHash'] = txHash
To2From = Relationship(To_Account, "traded", From_Account)
To2From['TxHash'] = txHash
self.graph.create(Tx)
self.graph.create(Tx2TxBlock)
self.graph.create(TxBlock2Tx)
# 若没在数据库中
if not existed_from:
self.graph.create(From_Account)
if not existed_to:
self.graph.create(To_Account)
self.graph.create(From2Tx)
self.graph.create(Tx2From)
self.graph.create(To2Tx)
self.graph.create(Tx2To)
self.graph.create(From2To)
self.graph.create(To2From)
def getAccountData(self, address_old, address_new):
# 当账户已经在数据库中时：
Account = self.matcher_node.match("Account", address=address_old).first()
res = self.getResult("GetBalance", address_old)
if Account:
# self.printf("account %s existed!" % address)
if "error" in res:
Account.update({"balance": "0", "existed": "false"})
else:
Account.update({"balance": res["balance"], "existed": "true"})
self.graph.push(Account)
return Account, 1
Account = Node("Account", address=address_old)
Account.update({"address_new": address_new})
# 获取账户当前的余额
if "error" in res:
Account.update({"balance": "0", "existed": "false"})
else:
Account.update({"balance": res["balance"], "existed": "true"})
return Account, 0
def run(self):
# 先对DSBlock进行存储
# self.getDsBlockData()
# 在对TxBlock存储的过程中将所有的交易信息
# 以及交易过程中的账户信息一并存储到数据库
curTxBlock = self.matcher_node.match("TxBlock", BlockNum=str(self.curTxBlockNum)).first()
if curTxBlock: self.graph.delete(curTxBlock)
try:
while True:
self.getOneTxBlcokData(str(self.curTxBlockNum))
self.curTxBlockNum += 1
except FunctionTimedOut:
self.printf("ERROR: Running Timeout: Restart!")
self.run()
except Exception as e:
self.printf("ERROR: %s: Restart!" % str(e))
self.run()
for entity in entities:
if entity in data[i]['Summary'].lower():
node = graph.evaluate('MATCH(e:Entity {name: "' + entity + '"}) RETURN e')
if not node:
node = graph.evaluate('CREATE(e:Entity {name: "' + entity + '"}) RETURN e')
# Create relationship of story with an entity
r = Relationship(node, "HAS_STORY", summary)
graph.create(r)
flag = True
# No entity related to story, link to another story
if not flag:
# Delete the previous Story node
graph.delete(summary)
# Create a "Substory" node of the summary instead
summary = Node("Substory", Topic=data[i]['topic']
, Summary=data[i]['Summary']
, Sentiment=data[i]['Sentiment']
, Date=data[i]['Date'])
graph.create(summary)
# Extract related stories
query = "MATCH(n:Story) WHERE "
for word in keyword.split():
query += "toLower(n.Summary) CONTAINS '" + word + "'" + " AND "
query = query.rstrip(' AND ')
class TwitterGraph():
def __init__(self):
self.graph = Graph("http://*****:*****@54.191.171.209:7474/db/data/")
self.popularity_heap = []
self.reassess_popularity()
def add_user(self, user):
new_user = Node("User", token=user.token.session_id, user_id=user.id)
return self.graph.create(new_user)
def is_cached(self, screen_name):
twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name)
if twitter_user is not None:
return True
def get_RT_recommendations(self, user):
recommendations = Counter()
user_node = self.graph.find_one("User", 'user_id', user.id)
following = user_node.match_outgoing("FOLLOWS", limit=5)
for rel in following:
retweets = rel.end_node.match_outgoing("RETWEETED", limit=5)
for r in retweets:
recommendations[r.end_node.properties['screen_name']] += 1
return [str for (str, count) in recommendations.most_common(10)]
def get_generic_recommendations(self):
return [screen_name for (count, screen_name) in heapq.nlargest(10, self.popularity_heap)]
def reassess_popularity(self):
# NOTE: expensive calculation, to be run threaded at multiples of x actions to graph or hourly/daily job
all_twitter_users = self.graph.find("TwitterUser")
for tu in all_twitter_users:
incoming_count = sum(1 for _ in tu.match_incoming())
heapq.heappush(self.popularity_heap, (incoming_count, tu.properties['screen_name']))
def add_twitter_user(self, screen_name):
twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name)
if twitter_user is None:
new_twitter_user = Node("TwitterUser", screen_name=screen_name)
self.graph.create(new_twitter_user)
def add_follow(self, screen_name, user):
user_node = self.graph.find_one("User", 'user_id', user.id)
if user_node is None:
# this shouldn't happen, just for testing while transitioning db
self.add_user(user)
user_node = self.graph.find_one("User", 'user_id', user.id)
twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name)
if twitter_user is None:
# this shouldn't happen, just for testing while transitioning db
self.add_twitter_user(screen_name)
twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name)
follow_relationship = Relationship(user_node, "FOLLOWS", twitter_user)
self.graph.create(follow_relationship)
self.reassess_popularity()
def remove_follow(self, screen_name, user):
user_node = self.graph.find_one("User", 'user_id', user.id)
if user_node is None:
# this shouldn't happen, just for testing while transitioning db
self.add_user(user)
user_node = self.graph.find_one("User", 'user_id', user.id)
twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name)
if twitter_user is None:
# this shouldn't happen, just for testing while transitioning db
self.add_twitter_user(screen_name)
twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name)
follow_relationship = self.graph.match_one(user_node, "FOLLOWS", twitter_user)
if follow_relationship is not None:
self.graph.delete(follow_relationship)
def add_retweet(self, screen_name, retweeted_screen_name):
twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name)
if twitter_user is None:
# this shouldn't happen, just for testing while transitioning db
self.add_twitter_user(screen_name)
twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name)
self.add_twitter_user(retweeted_screen_name)
retweeted_twitter_user = self.graph.find_one("TwitterUser", 'screen_name', retweeted_screen_name)
retweet = self.graph.match_one(twitter_user, "RETWEETED", retweeted_twitter_user)
if retweet is None:
retweet_relationship = Relationship(twitter_user, "RETWEETED", retweeted_twitter_user)
retweet_relationship.properties['count'] = 1
self.graph.create(retweet_relationship)
elif retweet.properties['count'] is None:
# this shouldn't happen, just for testing while transitioning db
retweet.properties['count'] = 1
retweet.push()
else:
retweet.properties['count'] = retweet.properties['count'] + 1
retweet.push()
# #START：更新节点
# node2 = graph.nodes[1733]
# node2['stockID'] = '001'
# node2['别称'] = '临时称呼'
# print(graph.nodes[1733])
#
# data = {
# 'stockID': '002',
# '别称': '惠州TCL'
# }
# node2.update(data)
# graph.nodes[1733]._set_attr('别称','临时称呼')
# print(graph.nodes[1733])
#END：更新节点
"""
#Start：删除节点后，关系自动删除了
# node1 = graph.nodes[771]
node2 = graph.nodes[1713]
# relationship = graph.match_one(rel_type='KNOWS')
# graph.delete(relationship)
graph.delete(node2)
#END：删除节点
"""
"""
#START:运行Neo4j命令
data = graph.run('MATCH (p:产品) RETURN p LIMIT 5')
print(list(data))
#END:运行Neo4j命令
"""
class GraphAccessor():
## tools 类型
#EntityType = 'Person'
## 匹配Node使用的key名称
#MatchKey = 'name'
def __init__(self):
self.graph = Graph(host='localhost', port=7687, password='')
self.nodeMatcher = NodeMatcher(self.graph)
'''
根据 entity_type, key, value 匹配获得对应的 Node.
@return:
'''
def matchNode(self, entity_type, key, value):
return self.graph.nodes.match(entity_type).where(
' _.{} = "{}" '.format(key, value))
#return self.nodeMatcher.match(self.graph).where(' _.{} = "{}" '.format(key, value))
'''
根据输入Entity获得对应的Node映射结果，返回Node对象.
'''
def nodeMap(self, node):
key = self.MatchKey
value = node[key]
node_list = self.matchNode(key, value)
if node_list.__len__() > 0:
## 源Entity -> Neo4j Entity映射计算，获得对应的Entity ID
return node_list.first()
else:
## 如果Neo4j 暂无当前Entity，则添加新的Node
return self.addNode(node)
return None
'''
根据uuid查询获得对应的Node.
'''
def findNode(self, uuid):
this_node = self.graph.nodes.match(self.EntityType).where(
' _.{} = "{}" '.format('uuid', uuid))
if not this_node:
return None
if this_node.__len__() != 1:
print sys.stderr << 'multiple node has same uuid.'
return None
return this_node.first()
'''
根据 entity_type, key, value 匹配获得对应的 Node.
'''
def findNodes(self, entity_type, key, value):
this_node = self.graph.nodes.match(entity_type).where(
' _.{} = "{}" '.format(key, value))
return this_node
def findNodeOne(self, entity_type, key, value):
this_node = self.graph.nodes.match(entity_type).where(
' _.{} = "{}" '.format(key, value))
assert len(this_node) <= 1
return this_node.first()
'''
Graph添加新的node, 返回node对象。
'''
def addNode(self, node):
self.graph.create(node)
return self.findNode(node.__uuid__)
'''
根据开始、结束结点，查询获得对应的边，可能为空；不考虑边关系。
'''
def findRelation(self, start_node, end_node):
#return self.graph.relationships.match().first()
return self.graph.match_one(nodes=[start_node, end_node])
'''
根据开始、结束结点以及边关系，查询获得对应的边，可能为空。
'''
def findRelationOne(self, start_node, end_node, r_type):
#return self.graph.relationships.match().first()
return self.graph.match_one(nodes=[start_node, end_node],
r_type=r_type)
'''
删除NodeMatch下所有Node
'''
def deleteNodes(self, nodes):
try:
for n in nodes:
self.graph.delete(n)
except TypeError:
try:
self.graph.delete(nodes)
except TypeError:
print "deleteNodes nodes type error."
def displyNodes(self, nodes):
for n in nodes:
print json.dumps(n, ensure_ascii=False)
class Network():
def __init__(self):
self.graph_instance = Graph()
self.time = self.update_time(str(datetime.datetime.now()))
# Updates current instance of time by system
def update_time(self, time):
self.time = time
# Checks if node exists, if it does not exist - creates new node; else, updates node
def add_node(self, link, date_last_updated, frequency):
calculated_frequency = convert_frequency_to_hours(frequency)
if (not self.check_node_exist(link)):
# Create a new node for webpage with an initial calculated frequency
n = Node(link,
date_last_updated=date_last_updated,
frequency=frequency,
calculated_frequency=calculated_frequency,
link=link)
self.graph_instance.create(n)
else:
# Update existing fields for webpage node
n = self.graph_instance.find_one(link)
if (n["date_last_updated"] != ""):
calculated_frequency = self._update_calculated_frequency(
n["date_last_updated"], date_last_updated)
n["date_last_updated"] = date_last_updated
n["calculated_frequency"] = calculated_frequency
n["frequency"] = frequency
n.push()
return n
# Measures calculated frequency from subtracting previous date_last_updated to current date_last_updated (returns time in hours)
def _update_calculated_frequency(self, prev_date_updated,
new_date_updated):
try:
prev_date = datetime.datetime.strptime(prev_date_updated,
"%Y-%m-%d")
new_date = datetime.datetime.strptime(new_date_updated, "%Y-%m-%d")
td = new_date - prev_date
return td.total_seconds() // 3600
except:
return -1
# If the relationship doesn't exist, create a new edge; else, update the tag
def add_edge(self, node_u, node_v_link, relationship):
self.add_node(node_v_link, "", "")
node_v = self.get_node(node_v_link)
self.graph_instance.create(
Relationship(node_u, "links_to", node_v, tag=relationship))
def check_node_exist(self, link):
return len(list(self.graph_instance.find(link))) != 0
def check_relationship_exist(self, node_u, node_v):
return len(
list(
self.graph_instance.match(start_node=node_u,
end_node=node_v,
rel_type="links_to"))) > 0
def delete_failed_webpages(self, link):
if (self.check_node_exist(link) == False):
return
node = self.get_node(link)
self.delete_relationship(node)
self.delete_incoming_relationship(node)
self.graph_instance.delete(node)
def delete_relationship(self, node_u):
rels = list(
self.graph_instance.match(rel_type="links_to",
start_node=node_u,
end_node=None))
for r in rels:
self.graph_instance.separate(r)
def delete_incoming_relationship(self, node_u):
rels = list(
self.graph_instance.match(rel_type="links_to",
end_node=node_u,
start_node=None))
for r in rels:
self.graph_instance.separate(r)
def get_node(self, link):
return self.graph_instance.find_one(link)
def get_node_information(self, link):
check_node = self.graph_instance.data("MATCH (n {link: '" + link +
"'} ) RETURN n")
if len(check_node) == 0:
return {}
n = self.get_node(link)
date_last_updated = n["date_last_updated"]
calculated_frequency = n["calculated_frequency"]
frequency = n["frequency"]
node_data = {}
node_data["date_last_updated"] = date_last_updated
node_data["calculated_frequency"] = calculated_frequency
node_data["frequency"] = frequency
node_data["outlinks"] = self.get_outlinks(link)
node_data["inlinks"] = self.get_inlinks(link)
return node_data
def get_outlinks(self, link):
outlink_data = self.graph_instance.data("MATCH (n {link: '" + link +
"'} )-->(node) RETURN node")
outlinks = []
for o in outlink_data:
outlinks.append(o["node"]["link"])
return outlinks
def get_inlinks(self, link):
inlink_data = self.graph_instance.data("MATCH (n {link: '" + link +
"'} )<--(node) RETURN node")
inlinks = []
for o in inlink_data:
inlinks.append(o["node"]["link"])
return inlinks
# Get adjacency matrix from Neo4j and nodes from py2neo
def _to_matrix(self):
nodes = list(self.graph_instance.node_selector.select())
N = len(nodes)
mat = np.zeros((N, N))
# Populate the adjacency matrix
for i, a in enumerate(nodes):
for j, b in enumerate(nodes):
# Use existing function to check for link
mat[i, j] = self.check_relationship_exist(a, b)
return mat
#Iterate over nodes and add pagerank
def update_pagerank(self):
# Get all the nodes
nodes = self.graph_instance.node_selector.select()
# Iterate over the result of _pagerank and the nodes
for pr, node in zip(self._pagerank(), nodes):
# Update the node's pagerank and push back to neo4j
node.update(page_rank=pr)
self.graph_instance.push(node)
# Simple show function to get nodes and display their pagerank
def show_pagerank(self, selector=None, link=None):
nodes = list(self.graph_instance.node_selector.select())
for node in nodes:
if isinstance(link, str):
if not list(node.labels())[0] == link:
continue
elif isinstance(link, (list, tuple)):
if not list(node.labels())[0] in link:
continue
# Get the pageranks for any given list of links (or all)
def get_pagerank_dict(self, links=[]):
nodes = list(self.graph_instance.node_selector.select())
dct = {}
for node in nodes:
if isinstance(links, str):
if not list(node.labels())[0] == links:
continue
elif isinstance(links, (list, tuple)):
if not list(node.labels())[0] in links:
continue
dct[list(node.labels())[0]] = node.get('page_rank')
return dct
# Creates dictionary object with information for ranking API (including page rank)
def get_ranking_data(self, links):
page_ranks = self.get_pagerank_dict(links)
data = {}
data["webpages"] = []
for l in page_ranks.keys():
webpage_data = {}
# If the node exists
if (page_ranks[l] != None):
n = self.get_node(l)
webpage_data["pageRankValue"] = page_ranks[l]
webpage_data["dateLastUpdated"] = n["date_last_updated"]
webpage_data["frequency"] = n["frequency"]
webpage_data["webpage"] = l
else:
webpage_data["pageRankValue"] = "NULL"
webpage_data["dateLastUpdated"] = ""
webpage_data["frequency"] = ""
webpage_data["webpage"] = ""
data["webpages"].append(webpage_data)
return data
# Perform pagerank on the adjacency matrix, using the power method
def _pagerank(
self,
alpha=0.85,
max_iter=100, # Increase this if we get the non-convergence error
tol=1.0e-6,
):
# Create a sparse matrix rep. of adjacency matrix
mat = scipy.sparse.csr_matrix(self._to_matrix())
n, m = mat.shape
# Make a sum matrix
S = scipy.array(mat.sum(axis=1)).flatten()
# Get non-zero rows
index = scipy.where(S <> 0)[0]
for i in index:
# We need to normlize (divide by sum)
mat[i, :] *= 1.0 / S[i]
#
pr = scipy.ones((n)) / n # initial guess
# Get dangling nodes
dangling = scipy.array(scipy.where(mat.sum(axis=1) == 0, 1.0 / n,
0)).flatten()
for i in range(max_iter):
prlast = pr
pr = alpha * (pr * mat + scipy.dot(dangling, prlast)) + (
1 - alpha) * prlast.sum() / n
# check if we're done
err = scipy.absolute(pr - prlast).sum()
if err < n * tol:
return pr
raise Exception("pagerank failed to converge [%d iterations]" %
(i + 1))
# Prioritizer
def prioritizer(self, outlinks):
# Get remaining time and number of inlink
for ol in outlinks:
if (not self.check_node_exist(ol)):
outlinks.remove(ol)
else:
self.remaining_time(ol)
self.sort_node(outlinks)
new_links = sorted(
outlinks,
key=lambda k:
(self.get_node(k)["time_remaining"], self.number_of_inlinks(k)))
for ol in new_links:
# Update last_crawled_time
current = str(datetime.datetime.now())
node = self.get_node(ol)
node["last_crawled_time"] = current
node.push()
return new_links
# Get number of inlink
def number_of_inlinks(self, outlink):
node = self.get_node(outlink)
return -len(
list(
self.graph_instance.match(
rel_type="links_to", end_node=node, start_node=None)))
# Updates remaining time left for a node to be crawled based on frequency
def remaining_time(self, outlink):
node = self.get_node(outlink)
last_crawled_time = node["last_crawled_time"]
if (last_crawled_time == None):
node["time_remaining"] = 0
node.push()
else:
fmt = '%Y-%m-%d %H:%M:%S'
current = str(datetime.datetime.now())
start = datetime.datetime.strptime(current[:19], fmt)
end = datetime.datetime.strptime(last_crawled_time[:19], fmt)
diff = (start - end).total_seconds() / 60.000 / 60.000
diff = float(node["calculated_frequency"]) - diff
node["time_remaining"] = diff
node.push()
# Sort node and fill top 100
def sort_node(self, outlinks):
num = len(outlinks)
count = 0
nodes = self.graph_instance.data("MATCH (n) RETURN n")
for n in nodes:
if (not n["n"]["link"] in outlinks):
self.remaining_time(n["n"]["link"])
nodes = self.graph_instance.data(
"MATCH (n) RETURN n ORDER BY (n.time_remaining) DESC")
for n in nodes:
link = n["n"]["link"]
if (not link in outlinks):
outlinks.append(link)
count = count + 1
if (count + num > 100):
break
# Return dictionary object of prioritized links and their priority value
def prioritize_dic(self, outlinks):
new_links = self.prioritizer(outlinks)
data = {}
data["prioritizedLinks"] = []
p_value = 1
for l in new_links:
l_data = {}
l_data["link"] = l
l_data["priority_value"] = p_value * 10
data["prioritizedLinks"].append(l_data)
p_value = p_value + 1
return data
# has_contrast = models.Relationship(Contrast,rel_type='HASCONTRAST')
for row in conditions.iterrows():
name = row[1].condition_text
user = row[1].id_user
uid = row[1].id
task = row[1].id_term
description = row[1].condition_description
timestamp = row[1].event_stamp
if not str(name) == "nan":
properties = {"description": description}
node = make_node("condition", uid, name, properties)
tasknode = find_node("task", property_value=task)
# If the tasknode is node found, then we do not create the relation, and we delete the condition
if tasknode == None:
graph.delete(node)
else:
relation = make_relation(tasknode, "HASCONDITION", node)
#class Disorder(models.NodeModel):
# has_difference = models.Relationship(Contrast,rel_type='HASDIFFERENCE')
# mentioned_in = models.Relationship('PMID',rel_type='MENTIONEDIN')
disorders.columns = ["id", "term",
"classification"] # Bug in returning column index name
for row in disorders.iterrows():
uid = row[1].id
classification = row[1].classification
name = row[1].term
properties = {"classification": classification}
node = make_node("disorder", uid, name, properties)
test_node_2 = Node("人", name="小红")
test_graph.create(test_node_1)
test_graph.create(test_node_2)
# 建立关系
node_1_call_node_2 = Relationship(test_node_1, '喜欢', test_node_2)
# node_1_call_node_2['count'] = 1
node_2_call_node_1 = Relationship(test_node_2, '讨厌', test_node_1)
# node_2_call_node_1['count'] = 2
test_graph.create(node_1_call_node_2)
test_graph.create(node_2_call_node_1)
# 删
test_node_3 = Node("人", name="小刚")
test_graph.create(test_node_3)
test_graph.delete(test_node_3)
# 改
test_node_2['age'] = 20
test_graph.push(test_node_2)
test_node_2['age'] = 22
test_graph.push(test_node_2)
node_1_call_node_2['程度'] = '非常'
test_graph.push(node_1_call_node_2)
# 查
data1 = test_graph.run('MATCH (a:人) RETURN a')
data = test_graph.run('MATCH (a:人) RETURN a').data()
