def create_person(node_type, node_key, node_id, node_name_index):
Index = ManualIndexManager(graph)
node_name = Index.get_or_create_index(Node, node_name_index)
if node_name:
exist = node_name.get(node_key, node_id)
if exist:
return 'Node Exist' #节点已存在
else:
if node_key == "uid":
person_node = Node(node_type, uid=node_id)
elif node_key == "org_id":
person_node = Node(node_type, org_id=node_id)
elif node_key == "event_id":
person_node = Node(node_type, event_id=node_id)
elif node_key == "event":
person_node = Node(node_type, event=node_id)
elif node_key == "group":
person_node = Node(node_type, group=node_id)
else:
return 'Node Type Wrong' #节点类型错误
graph.create(person_node)
node_name.add(node_key, node_id, person_node)
return 'Node Success' #创建节点成功
else:
return 'Node Wrong' #创建节点失败
def put_in_node():
Index = ManualIndexManager(graph) # manage index
tx = graph.begin()
count = 0
index_list = [] # temporal put in
filter_set = set()
ts = time.time()
#domain_name = "domain_index"
domain_name = "topic_index"
domain_index = Index.get_or_create_index(Node, domain_name)
Batch_index = ManualIndexWriteBatch(graph)
#for item in domain_list:
for item in topic_list:
#st_node = Index.get_or_create_indexed_node(node_name, "uid", start_node, {"uid":start_node})
#ed_node = Index.get_or_create_indexed_node(node_name, "uid", end_node, {"uid":end_node})
#exist = Index.get_indexed_node(domain_name, "domain", item)
exist = Index.get_indexed_node(domain_name, "topic", item)
if not exist:
#node = Node("Domain", domain=item)
node = Node("Topic", topic=item)
tx.create(node)
index_list.append([item, node])
tx.commit()
for item in index_list:
#domain_index.add('domain', item[0], item[1])
domain_index.add('topic', item[0], item[1])
print domain_index
print domain_index.get("topic", '科技类')
def create_node_location():
Index = ManualIndexManager(graph)
node_index = Index.get_index(Node, node_index_name)
location_index = Index.get_or_create_index(Node, location_index_name)
f = open("user_portrait.txt", "rb")
count = 0
update_node = []
index_list = []
in_set = set()
tx = graph.begin()
for item in f:
user_dict = json.loads(item)
each_location = user_dict["location"]
exist = location_index.get("location", each_location)
if not exist and each_location not in in_set:
node = Node("Location", location=each_location)
tx.create(node)
index_list.append([each_location, node])
count += 1
in_set.add(each_location)
if count % 10 == 0:
print count
tx.commit()
tx = graph.begin()
for item in index_list:
location_index.add("location", item[0], item[1])
index_list = []
tx.commit()
if index_list:
for item in index_list:
location_index.add("location", item[0], item[1])
print "all done"
f.close()
def create_node_and_rel(node_key1, node1_list, node1_index_name, rel, node_key2, \
node2_id, node2_index_name, submit_user, k_label, node2_name):
Index = ManualIndexManager(graph) # manage index
group_index = Index.get_or_create_index(Node, node2_index_name)
p_node2_id = p.get_pinyin(node2_id)
p_node2_id = p_node2_id.lower()
c_string = "START end_node=node:%s(%s='%s') RETURN end_node"\
% (node2_index_name, node_key2, p_node2_id)
print c_string
try:
result = graph.run(c_string)
except:
result = []
node_l = []
for i in result:
node_l.append(i[0])
if len(node_l) > 0: #判断对否有该节点存在
return 'group already exist'
else:
group_dict = {}
group_dict['group_name'] = node2_name
group_dict['people'] = '&'.join(node1_list)
group_dict['people_count'] = len(node1_list)
group_dict['create_ts'] = int(time.time())
group_dict['user'] = submit_user
group_dict['k_label'] = '&'.join(k_label.split(','))
group_id = p.get_pinyin(node2_id)
group_id = group_id.lower()
labels = get_special_labels(node1_list)
group_dict['label'] = labels
wiki_link = getUrlByKeyWordList(labels)
group_dict['wiki_link'] = json.dumps(wiki_link)
# es_group.delete(index=group_name, doc_type=group_type, id='*****@*****.**')
es_group.index(index=group_name,
doc_type=group_type,
id=group_id,
body=group_dict)
new_group = Node(group_node, group=group_id)
graph.create(new_group)
group_index.add("group", group_id, new_group)
# return 'succeed'
user_org = search_user_type(node1_list)
user_id = user_org[0]
org_id = user_org[1]
flag = create_rel(node_key1, user_id, node1_index_name, rel, node_key2,
group_id, node2_index_name, submit_user)
node_key11 = org_primary
node11_index_name = org_index_name
info = create_rel(node_key1, org_id, node1_index_name, rel, node_key2,
group_id, node2_index_name, submit_user)
return info
def put_in_special_event(event_name):
Index = ManualIndexManager(graph)
special_event_index = Index.get_or_create_index(Node,
special_event_index_name)
exist = special_event_index.get("event", event_name)
if not exist:
node = Node("SpecialEvent", event=event_name)
tx = graph.begin()
tx.create(node)
tx.commit()
special_event_index.add("event", event_name, node)
return 1
return 0
def create_node_and_rel(node_key1, node1_list, node1_index_name, rel, node_key2, \
node2_id, node2_index_name, submit_user, k_label, node2_name):
Index = ManualIndexManager(graph) # manage index
theme_index = Index.get_or_create_index(Node, node2_index_name)
p_node2_id = p.get_pinyin(node2_id)
p_node2_id = p_node2_id.lower()
c_string = "START end_node=node:%s(%s='%s') RETURN end_node"\
% (node2_index_name, node_key2, p_node2_id)
print c_string
try:
result = graph.run(c_string)
except:
result = []
node_l = []
for i in result:
# node1_l
node_l.append(i[0])
if len(node_l) > 0: #判断对否有该节点存在
return 'theme already exist'
else:
theme_dict = {}
theme_dict['topic_name'] = node2_name
theme_dict['event'] = '&'.join(node1_list)
theme_dict['event_count'] = len(node1_list)
theme_dict['create_ts'] = int(time.time())
theme_dict['user'] = submit_user
if k_label:
k_label = '&'.join(k_label.split(','))
theme_dict['k_label'] = k_label
topic_id = p.get_pinyin(node2_id)
topic_id = topic_id.lower()
labels = get_special_labels(node1_list)
theme_dict['label'] = labels
wiki_link = getUrlByKeyWordList(labels)
theme_dict['wiki_link'] = json.dumps(wiki_link)
# es_event.delete(index=special_event_name, doc_type=special_event_type, id='*****@*****.**')
es_event.index(index=special_event_name,
doc_type=special_event_type,
id=topic_id,
body=theme_dict)
new_theme = Node(special_event_node, event=topic_id)
graph.create(new_theme)
theme_index.add("event", topic_id, new_theme)
# return 'succeed'
info = create_rel(node_key1, node1_list, node1_index_name, rel, node_key2,
topic_id, node2_index_name, submit_user)
return info
def create_tag(tag_id, attribute_dict=dict()):
Index = ManualIndexManager(graph) # manage index
index_name = "tag_index"
tag_index = Index.get_or_create_index(Node, index_name)
exist = tag_index.get("tag", tag_id)
if exist:
tag_node = exist[0]
for k, v in attribute_dict.iteritems():
tag_node[k] = v
graph.push(tag_node)
else:
tag_node = Node("Tag",
tag=tag_id) # create event node with only one event_id
for k, v in attribute_dict.iteritems():
tag_node[k] = v
graph.create(tag_node)
tag_index.add("tag", tag_id, tag_node)
return True
def create_event(event, attribute_dict=dict()):
Index = ManualIndexManager(graph) # manage index
event_index = Index.get_or_create_index(Node, event_index_name)
exist = event_index.get("event", event)
if exist:
event_node = exist[0]
for k, v in attribute_dict.iteritems():
event_node[k] = v
graph.push(event_node)
else:
event_node = Node(
"Event",
event_id=event) # create event node with only one event_id
for k, v in attribute_dict.iteritems():
event_node[k] = v
graph.create(event_node)
event_index.add("event", event, event_node)
return True
def create_group(group_id, attribute_dict=dict()):
Index = ManualIndexManager(graph) # manage index
group_index = Index.get_or_create_index(Node, group_index_name)
exist = group_index.get("group", group_id)
if exist:
group_node = exist[0]
for k, v in attribute_dict.iteritems():
group_node[k] = v
graph.push(group_node)
else:
group_node = Node(
"Group",
group=group_id) # create event node with only one event_id
for k, v in attribute_dict.iteritems():
group_node[k] = v
graph.create(group_node)
group_index.add("group", group_id, group_node)
return True
def put_in():
Index = ManualIndexManager(graph) # manage index
tx = graph.begin()
count = 0
index_list = [] # temporal put in
filter_set = set()
ts = time.time()
node_name = "node_index"
#rel_name = "rel_index"
node_index = Index.get_or_create_index(Node, node_name)
#rel_index = Index.get_or_create_index(Relationship, rel_name)
Batch_index = ManualIndexWriteBatch(graph)
user_list = json.loads(r.get("user_set"))
for user in user_list:
#st_node = Index.get_or_create_indexed_node(node_name, "uid", start_node, {"uid":start_node})
#ed_node = Index.get_or_create_indexed_node(node_name, "uid", end_node, {"uid":end_node})
exist = Index.get_indexed_node(node_name, "uid", user)
if not exist and user not in filter_set:
node = Node("User", uid=user)
tx.create(node)
index_list.append([user, node])
filter_set.add(node)
count += 1
if count % 1000 == 0:
print count
te = time.time()
print "cost time: %s" % (te - ts)
ts = te
tx.commit()
tx = graph.begin()
for item in index_list:
Batch_index.add_to_index(Node, node_name, "uid", item[0],
item[1])
index_list = []
filter_set = set()
tx.commit()
def create_person(node_type,
node_key,
node_id,
node_name_index,
attribute_dict=dict()):
Index = ManualIndexManager(graph)
node_name = Index.get_or_create_index(Node, node_name_index)
print node_name
if node_name:
exist = node_name.get(node_key, node_id)
print exist
if exist:
person_node = exist[0]
for k, v in attribute_dict.iteritems():
person_node[k] = v
graph.push(person_node)
else:
person_node = Node(node_type, uid=node_id)
for k, v in attribute_dict.iteritems():
person_node[k] = v
graph.create(person_node)
node_name.add(node_key, node_id, person_node)
print "create person success"
return True
class Store(object):
""" Virtual storage mapped onto an existing graph in which
objects can be stored.
"""
def __init__(self, graph):
self.graph = graph
self.index_manager = ManualIndexManager(self.graph)
self.__delete_query = ("MATCH (a) WHERE id(a)={A} "
"OPTIONAL MATCH a-[r]-b "
"DELETE r, a")
def _assert_saved(self, subj):
try:
node = subj.__node__
if node is None:
raise NotSaved(subj)
except AttributeError:
raise NotSaved(subj)
def _get_node(self, endpoint):
if isinstance(endpoint, Node):
return endpoint
if not hasattr(endpoint, "__node__"):
self.save(endpoint)
return endpoint.__node__
def _is_same(self, obj, endpoint):
if isinstance(endpoint, Node):
if hasattr(obj, "__node__"):
return endpoint == obj.__node__
else:
return False
else:
return endpoint is obj
def is_saved(self, subj):
""" Return :py:const:`True` if the object `subj` has been saved to
the database, :py:const:`False` otherwise.
:param subj: the object to test
"""
return hasattr(subj, "__node__") and subj.__node__ is not None
def relate(self, subj, rel_type, obj, properties=None):
""" Define a relationship between `subj` and `obj` of type `rel_type`.
This is a local operation only: nothing is saved to the database until
a save method is called. Relationship properties may optionally be
specified.
:param subj: the object bound to the start of the relationship
:param rel_type: the relationship type
:param obj: the object bound to the end of the relationship
:param properties: properties attached to the relationship (optional)
"""
if not hasattr(subj, "__rel__"):
subj.__rel__ = {}
if rel_type not in subj.__rel__:
subj.__rel__[rel_type] = []
subj.__rel__[rel_type].append((properties or {}, obj))
def separate(self, subj, rel_type, obj=None):
""" Remove any relationship definitions which match the criteria
specified. This is a local operation only: nothing is saved to the
database until a save method is called. If no object is specified, all
relationships of type `rel_type` are removed.
:param subj: the object bound to the start of the relationship
:param rel_type: the relationship type
:param obj: the object bound to the end of the relationship (optional)
"""
if not hasattr(subj, "__rel__"):
return
if rel_type not in subj.__rel__:
return
if obj is None:
del subj.__rel__[rel_type]
else:
subj.__rel__[rel_type] = [
(props, endpoint) for props, endpoint in subj.__rel__[rel_type]
if not self._is_same(obj, endpoint)
]
def load_related(self, subj, rel_type, cls):
""" Load all nodes related to `subj` by a relationship of type
`rel_type` into objects of type `cls`.
:param subj: the object bound to the start of the relationship
:param rel_type: the relationship type
:param cls: the class to load all related objects into
:return: list of `cls` instances
"""
if not hasattr(subj, "__rel__"):
return []
if rel_type not in subj.__rel__:
return []
return [
self.load(cls, self._get_node(endpoint))
for rel_props, endpoint in subj.__rel__[rel_type]
]
def load(self, cls, node):
""" Load and return an object of type `cls` from database node `node`.
:param cls: the class of the object to be returned
:param node: the node from which to load object data
:return: a `cls` instance
"""
subj = cls()
setattr(subj, "__node__", node)
self.reload(subj)
return subj
def load_indexed(self, index_name, key, value, cls):
""" Load zero or more indexed nodes from the database into a list of
objects.
:param index_name: the node index name
:param key: the index key
:param value: the index value
:param cls: the class of the object to be returned
:return: a list of `cls` instances
"""
index = self.index_manager.get_index(Node, index_name)
nodes = index.get(key, value)
return [self.load(cls, node) for node in nodes]
def load_unique(self, index_name, key, value, cls):
""" Load a uniquely indexed node from the database into an object.
:param index_name: the node index name
:param key: the index key
:param value: the index value
:param cls: the class of the object to be returned
:return: as instance of `cls` containing the loaded data
"""
index = self.index_manager.get_index(Node, index_name)
nodes = index.get(key, value)
if not nodes:
return None
if len(nodes) > 1:
raise LookupError("Multiple nodes match the given criteria; "
"consider using `load_all` instead.")
return self.load(cls, nodes[0])
def reload(self, subj):
""" Reload properties and relationships from a database node into
`subj`.
:param subj: the object to reload
:raise NotSaved: if `subj` is not linked to a database node
"""
self._assert_saved(subj)
# naively copy properties from node to object
properties = subj.__node__.properties
for key in subj.__dict__:
if not key.startswith("_") and key not in properties:
setattr(subj, key, None)
for key, value in properties.items():
if not key.startswith("_"):
setattr(subj, key, value)
subj.__rel__ = {}
for rel in subj.__node__.match():
if rel.type not in subj.__rel__:
subj.__rel__[rel.type] = []
subj.__rel__[rel.type].append((rel.properties, rel.end_node))
def save(self, subj, node=None):
""" Save an object to a database node.
:param subj: the object to save
:param node: the database node to save to (if omitted, will re-save to
same node as previous save)
"""
if node is not None:
subj.__node__ = node
# naively copy properties from object to node
props = {}
for key, value in subj.__dict__.items():
if not key.startswith("_"):
props[key] = value
if hasattr(subj, "__node__"):
subj.__node__.properties.clear()
subj.__node__.properties.update(props)
self.graph.cypher.execute(
"MATCH (a) WHERE id(a)={a} MATCH (a)-[r]->(b) DELETE r",
{"a": subj.__node__})
else:
subj.__node__, = self.graph.create(props)
# write rels
if hasattr(subj, "__rel__"):
batch = WriteBatch(self.graph)
for rel_type, rels in subj.__rel__.items():
for rel_props, endpoint in rels:
end_node = self._get_node(endpoint)
if end_node not in self.graph:
raise ValueError(end_node)
batch.create(
(subj.__node__, rel_type, end_node, rel_props))
batch.run()
return subj
def save_indexed(self, index_name, key, value, *subj):
""" Save one or more objects to the database, indexed under the
supplied criteria.
:param index_name: the node index name
:param key: the index key
:param value: the index value
:param subj: one or more objects to save
"""
index = self.index_manager.get_or_create_index(Node, index_name)
for subj in subj:
index.add(key, value, self.save(self._get_node(subj)))
def save_unique(self, index_name, key, value, subj):
""" Save an object to the database, uniquely indexed under the
supplied criteria.
:param index_name: the node index name
:param key: the index key
:param value: the index value
:param subj: the object to save
"""
index = self.index_manager.get_or_create_index(Node, index_name)
node = index.get_or_create(key, value, {})
self.save(subj, node)
def delete(self, subj):
""" Delete a saved object node from the database as well as all
incoming and outgoing relationships.
:param subj: the object to delete from the database
:raise NotSaved: if `subj` is not linked to a database node
"""
self._assert_saved(subj)
node = subj.__node__
del subj.__node__
self.graph.cypher.execute(self.__delete_query, {"A": node})