def add_post(self, title, tags, text):
user = self.find()
post = Node(
'Post',
id=str(uuid.uuid4()),
title=title,
text=text,
timestamp=timestamp(),
date=date()
)
rel = Relationship(user, 'PUBLISHED', post)
graph.create(rel)
tags = [x.strip() for x in tags.lower().split(',')]
for name in set(tags):
tag = Node('Tag', name=name)
graph.merge(tag,"Tag", "name")
rel = Relationship(tag, 'TAGGED', post)
graph.create(rel)
def get_linked_ids(idlist, iterations=1):
target_id_nodes = []
linked_id_nodes = []
all_link_nodes = []
all_link_edges = []
idlist2 = []
for i in range(len(idlist)):
linker_handle = Entrez.elink(db="pubmed",
dbfrom="pubmed",
id=idlist[i],
rettype="xml")
linker_record = soup(linker_handle.read(), "xml")
thisId = Node("PMID",
name="id_{}".format(idlist[i]),
id=idlist[i],
year=get_details(idlist[i])[1],
title="id_{}".format(idlist[i]),
full_title=get_details(idlist[i])[0])
all_link_nodes.append(thisId)
linker_records = linker_record.findAll("Link")
linked_id_nodes_now = [
Node("PMID",
name="id_{}".format(linker_records[l].Id.text),
id=linker_records[l].Id.text,
year=get_details(linker_records[l].Id.text)[1],
title="id_{}".format(linker_records[l].Id.text),
full_title=get_details(linker_records[l].Id.text)[0])
for l in range(len(linker_records))
]
idlist2.extend(
list(set([l.Id.text for l in linker_record.findAll("Link")])))
all_link_nodes.extend(linked_id_nodes)
linked_to = [
Relationship(thisId, "LINKED_TO", thisNode)
for thisNode in linked_id_nodes_now
]
all_link_edges.extend(linked_to)
if iterations == 1:
return all_link_nodes, all_link_edges, idlist2
elif iterations > 1:
all_link_nodes = []
all_link_edges = []
idlist3 = []
for j in range(iterations):
print("[+] Iteration number: {}".format(j))
nodes, edges, idlist_j = get_linked_ids(idlist)
all_link_edges.extend(edges)
all_link_nodes.extend(nodes)
idlist3.extend(list(set(idlist_j)))
return all_link_nodes, all_link_edges, idlist3
def permlize_relationship_result(record_node: Relationship):
node_type = 'Unknown'
try:
node_type = list(record_node.types())[0]
except _:
print('Exception occured with getting node_type')
value_dict = dict(record_node)
value_dict["from"] = dict(record_node.start_node)['permID']
value_dict["to"] = dict(record_node.end_node)['permID']
value_dict["type"] = node_type
return value_dict
def test_can_iterate_path(self):
a = Node(name="Alice")
b = Node(name="Bob")
c = Node(name="Carol")
d = Node(name="Dave")
e = Node(name="Eve")
f = Node(name="Frank")
path = Path(a, "KNOWS", b, "KNOWS", c, "KNOWS", d, "KNOWS", e, "KNOWS",
f)
assert list(path) == [
Relationship(a, 'KNOWS', b),
Relationship(b, 'KNOWS', c),
Relationship(c, 'KNOWS', d),
Relationship(d, 'KNOWS', e),
Relationship(e, 'KNOWS', f),
]
assert list(enumerate(path)) == [(0, Relationship(a, 'KNOWS', b)),
(1, Relationship(b, 'KNOWS', c)),
(2, Relationship(c, 'KNOWS', d)),
(3, Relationship(d, 'KNOWS', e)),
(4, Relationship(e, 'KNOWS', f))]
def connect_users_cities(graph, args):
city_dict = graph_util.query_label_to_dict(graph, "City", "name")
users = graph.nodes.match("User")
relationships = []
print("Creating IS_FROM relationships..")
for user in tqdm(users):
for city in user['city']:
relationships.append(Relationship(user, "IS_FROM",
city_dict[city]))
print("Done.")
print("Writing %d relationships.. " % len(relationships))
graph_util.create_in_batch(graph, relationships)
print("Done.")
class RelationshipLoopTestCase(TestCase):
loop = Relationship(alice, "LIKES", alice)
def test_nodes(self):
nodes = self.loop.nodes
assert isinstance(nodes, tuple)
assert nodes == (alice, alice)
def test_relationships(self):
relationships = self.loop.relationships
assert isinstance(relationships, tuple)
assert relationships == (self.loop,)
def write_locations(graph, args):
#### and let's write the location community.
def create_state_node_dict(states):
result = {}
for state in set(states):
result[state] = Node("State", name=state)
return result
def load_city_mapping():
""" closed in function to get data
"""
city_mapping = {}
find_state = re.compile("(\(\w{2}\))")
find_city = re.compile("(.+ )")
with open("../data/german_cities_raw.txt", encoding='utf-8',
mode='r') as f:
for entry in f:
city = find_city.findall(entry)[0].strip()
state = find_state.findall(entry)[0].replace("(", "").replace(
")", "")
city_mapping[city] = state
return city_mapping
city_mapping = load_city_mapping()
cities = []
states = []
for user in graph.nodes.match("User"):
cities.extend(user['city'])
states.extend(user['state'])
states = create_state_node_dict(states)
print("Writing state nodes..")
graph_util.create_in_batch(graph, list(states.values()))
print("Done.")
city_nodes = []
for city in set(cities):
city_nodes.append(Node("City", name=city))
print("Creating city nodes..")
graph_util.create_in_batch(graph, city_nodes)
print("Done.")
city_nodes
city_to_states = []
for city_node in city_nodes:
city_to_states.append(
Relationship(city_node, "IS_IN",
states[city_mapping[city_node['name']]]))
print("Creating city to state relationships..")
graph_util.create_in_batch(graph, city_to_states)
print("Done.")
def build_relationships(session,
graph,
orm,
data_row,
rel_name,
parent_orm=None):
"""Build a py2neo.Relationship object from SqlAlchemy objects.x
Args:
session (sqlalchemy.Session): SQL DB session.
transaction (py2neo.Transaction): Neo4j transaction
orm (sqlalchemy.Base): A SqlAlchemy ORM
rel_name (str): Name of the relationship to be added to Neo4j
parent_orm (sqlalchemy.Base): Another ORM to build relationship to.
If this is not specified, it implies
that :obj:`orm` is node, rather than a
relationship.
Returns:
{relationship, back_relationship}: Relationships pointing to the node (inferred from ORM), and one pointing back to it's associated project.
"""
# Case 1) the ORM represents a node, since it has no parents
if parent_orm is None:
this_node = Node(_extract_name(orm.__tablename__), **data_row)
rel_props = {}
# Case 2) this ORM represents a relationship, pointing to a parent
else:
this_node = retrieve_node(session, graph, orm, parent_orm, data_row)
rel_props = data_row
# Also retrieve the Project node from neo4j
proj_node = retrieve_node(session, graph, orm, Project, data_row)
# If either node is not found, give up
if proj_node is None or this_node is None:
return None, None
# Build a forward and backward relationship, wrt the project
relationship = Relationship(proj_node, rel_name, this_node, **rel_props)
back_relationship = Relationship(this_node, 'HAS_PROJECT', proj_node)
return relationship, back_relationship
def connect_tweets_users(graph, args):
relationships = []
user_nodes = graph_util.query_label_to_dict(graph, "User", "screen_name")
tweet_nodes = graph.nodes.match("Tweet")
print("Creating relationships between users via tweets..")
for tweet in tqdm(tweet_nodes):
if tweet['screen_name'] in user_nodes.keys():
user = user_nodes[tweet['screen_name']] # relate the tweet
relationships.append(Relationship(user, "TWEETED", tweet))
if isinstance(tweet['mentions'], list):
for mention in tweet['mentions']:
if mention in user_nodes.keys():
mentioned_user = user_nodes[
mention] # grabbing the user object by screen_name
relationships.append(
Relationship(tweet, "MENTIONS", mentioned_user))
print("Done.")
#### lets write mentions
print("Writing %d relationships to database.." % len(relationships))
graph_util.create_in_batch(graph, relationships)
print("Done.")
def create_relationship_in_gics_node(graph: Graph):
t = graph.begin()
n0 = graph.nodes.match("gics", "Sector")
n0_list = list(n0)
n1 = graph.nodes.match("gics", "Industry_Group")
n1_list = list(n1)
n2 = graph.nodes.match("gics", "Industry")
n2_list = list(n2)
n3 = graph.nodes.match("gics", "Sub_Industry")
n3_list = list(n3)
for nx in n0_list:
for ny in n1_list:
if re.match(nx['code'], ny['code']):
t.create(Relationship(nx, 'sub_class', ny))
for nx in n1_list:
for ny in n2_list:
if re.match(nx['code'], ny['code']):
t.create(Relationship(nx, 'sub_class', ny))
for nx in n2_list:
for ny in n3_list:
if re.match(nx['code'], ny['code']):
t.create(Relationship(nx, 'sub_class', ny))
t.commit()
def hydrate_object(obj, inst=None):
from py2neo.data import Node, Relationship, Path
from py2neo.client.packstream import Structure
if isinstance(obj, Structure):
tag = obj.tag
fields = obj.fields
if tag == ord(b"N"):
return Node.hydrate(self.graph, fields[0], fields[1], hydrate_object(fields[2]), into=inst)
elif tag == ord(b"R"):
return Relationship.hydrate(self.graph, fields[0],
fields[1], fields[2],
fields[3], hydrate_object(fields[4]), into=inst)
elif tag == ord(b"P"):
# Herein lies a dirty hack to retrieve missing relationship
# detail for paths received over HTTP.
nodes = [hydrate_object(node) for node in fields[0]]
u_rels = []
typeless_u_rel_ids = []
for r in fields[1]:
u_rel = self.unbound_relationship(*map(hydrate_object, r))
assert u_rel.type is None
typeless_u_rel_ids.append(u_rel.id)
u_rels.append(u_rel)
if typeless_u_rel_ids:
r_dict = {r.identity: r for r in RelationshipMatcher(graph).get(typeless_u_rel_ids)}
for i, u_rel in enumerate(u_rels):
if u_rel.type is None:
u_rels[i] = self.unbound_relationship(
u_rel.id,
type(r_dict[u_rel.id]).__name__,
u_rel.properties
)
sequence = fields[2]
return Path.hydrate(self.graph, nodes, u_rels, sequence)
else:
try:
f = self.hydration_functions[tag]
except KeyError:
# If we don't recognise the structure type, just return it as-is
return obj
else:
return f(*map(hydrate_object, obj.fields))
elif isinstance(obj, list):
return list(map(hydrate_object, obj))
elif isinstance(obj, dict):
return {key: hydrate_object(value) for key, value in obj.items()}
else:
return obj
def test_can_push_relationship(self):
a = Node()
b = Node()
ab = Relationship(a, "KNOWS", b)
self.graph.create(ab)
value = self.graph.evaluate(
"MATCH ()-[ab:KNOWS]->() WHERE id(ab)={i} "
"RETURN ab.since",
i=ab.identity)
assert value is None
ab["since"] = 1999
self.graph.push(ab)
value = self.graph.evaluate(
"MATCH ()-[ab:KNOWS]->() WHERE id(ab)={i} "
"RETURN ab.since",
i=ab.identity)
assert value == 1999
def instance_constructor():
from py2neo.data import Relationship
if properties is None:
new_instance = Relationship(self.hydrate_node(None, start), type,
self.hydrate_node(None, end))
new_instance._stale.add("properties")
else:
new_instance = Relationship(self.hydrate_node(None, start), type,
self.hydrate_node(None, end), **properties)
new_instance.graph = self.graph
new_instance.identity = identity
return new_instance
def insert_one_movie(self, node1, node2, rel_type, movie_id, movie_name):
# 关系
if node1 and node2:
rel = self.graph.match((node1, node2), rel_type).first()
if rel:
# 若存在关系，则添加电影名，电影名/id不能重复
if movie_id not in rel[MOVIE_ID]:
rel[MOVIE_ID].append(movie_id)
rel[MOVIE_NAME].append(movie_name)
self.graph.push(rel)
return
else:
# 若不存在，则新建一个关系
rel = Relationship(node1, rel_type, node2)
rel[MOVIE_ID] = [movie_id]
rel[MOVIE_NAME] = [movie_name]
self.graph.create(rel)
return
def test_can_merge_relationship_that_does_not_exist(self):
alice = Node("Person", name="Alice")
bob = Node("Person", name="Bob")
ab = Relationship(alice, "KNOWS", bob)
old_order = len(self.graph.nodes)
old_size = len(self.graph.relationships)
self.graph.merge(ab, "Person", "name")
self.assertEqual(alice.graph, self.graph)
self.assertIsNotNone(alice.identity)
self.assertEqual(bob.graph, self.graph)
self.assertIsNotNone(bob.identity)
self.assertEqual(ab.graph, self.graph)
self.assertIsNotNone(ab.identity)
assert self.graph.exists(alice | bob | ab)
new_order = len(self.graph.nodes)
new_size = len(self.graph.relationships)
assert new_order == old_order + 2
assert new_size == old_size + 1
def test_relationship_cache_is_thread_local(self):
import threading
a = Node()
b = Node()
r = Relationship(a, "TO", b)
self.graph.create(r)
assert r.identity in self.graph.relationship_cache
other_relationship_cache_keys = []
def check_cache():
other_relationship_cache_keys.extend(
self.graph.relationship_cache.keys())
thread = threading.Thread(target=check_cache)
thread.start()
thread.join()
assert r.identity in self.graph.relationship_cache
assert r.identity not in other_relationship_cache_keys
def add_relationship(self, start: Node, end: Node, kind: str,
props: dict) -> Relationship:
"""
Add a Relationship to database.
Parameters
------------
start: start Node
end: end Node
kind: Relationship kind
props: Relationship property
Returns
--------
out: a Relationship.
"""
relation = Relationship(start, kind, end, **props)
self.push_graph(relation)
return relation
def _gen_rel(self, rel_conf):
"""
根据关系的配置从source_neo4j 中找到对应的关系，添加到target_neo4j 中
:param rel_conf: {
"symmetrical": false,
"type": "HAS_ACTOR",
"startLabels": [
"Movie"
],
"transitive": false,
"endLabels": [
"Actor"
]
}
:return: set<Relationship> 新创建的关系（已被添加到target_neo4j 中）
"""
tx = self.target_neo4j.graph.begin()
step = 3000
idx = 0
rel_type = rel_conf['type']
start_labels = rel_conf['startLabels']
end_labels = rel_conf['endLabels']
rel_records = self.view_util.extract_rels(self.source_neo4j, rel_type,
start_labels, end_labels)
# 记录下新创建的关系，可能用于对称和传递的关系扩展
new_rel_set = set()
for record in rel_records:
new_start_node = self.node_map[record[self.START_ID]]
new_end_node = self.node_map[record[self.END_ID]]
new_rel = Relationship(new_start_node, rel_type, new_end_node)
tx.create(new_rel)
new_rel_set.add(new_rel)
idx += 1
if idx >= step:
idx = 0
tx.commit()
tx = self.target_neo4j.graph.begin()
tx.commit()
return new_rel_set
def annotate_politicians(graph, args):
with open("../data/politician_user_handles.json",
encoding='utf-8',
mode='r') as f:
inf = f.read()
temp = json.loads(inf)
parties = {}
politicians = {}
## flip the dict around to have things more readable
for party in temp.keys():
parties[party] = Node('Party',
name=party) ## create party nodes along the way
for politician in temp[party]:
politicians[politician] = party
relationships = []
graph_util.create_in_batch(graph, list(parties.values()))
print("Done.")
print("Creating party membership relations..")
for politician in tqdm(politicians.keys()):
politician_node = graph.nodes.match("User").where(
"_.screen_name = \"%s\"" % politician).first()
if politician_node != None:
### this needs to be done when updating anything, this is bullshit.
graph.merge(politician_node)
politician_node.add_label("Politician")
graph.push(politician_node)
relationships.append(
Relationship(politician_node, "IS_MEMBER_OF",
parties[politicians[politician]]))
print("Done.")
print("Writing %d party membership relations to database..")
graph_util.create_in_batch(graph, relationships)
print("Done.")
def test_relationship_property_pull_scenarios(self):
property_sets = [{}, {
"name": "Alice"
}, {
"name": "Alice",
"age": 33
}, {
"name": "Bob"
}]
for old_props in property_sets:
for new_props in property_sets:
a = Node()
b = Node()
relationship = Relationship(a, "TO", b, **old_props)
self.graph.create(relationship)
relationship_id = relationship.identity
assert dict(relationship) == old_props
self.graph.run("MATCH ()-[r]->() WHERE id(r)={x} SET r={y}",
x=relationship_id,
y=new_props)
self.graph.pull(relationship)
assert dict(relationship) == new_props, \
"Failed to pull new properties %r over old properties %r" % \
(new_props, old_props)
def test_path_in_several_ways(self):
alice = Node(name="Alice")
bob = Node(name="Bob")
carol = Node(name="Carol")
dave = Node(name="Dave")
path = Path(alice, "LOVES", bob, Relationship(carol, "HATES", bob),
carol, "KNOWS", dave)
assert path.__bool__()
assert path.__nonzero__()
assert path[0] == Relationship(alice, "LOVES", bob)
assert path[1] == Relationship(carol, "HATES", bob)
assert path[2] == Relationship(carol, "KNOWS", dave)
assert path[-1] == Relationship(carol, "KNOWS", dave)
assert path[0:1] == Path(alice, "LOVES", bob)
assert path[0:2] == Path(alice, "LOVES", bob,
Relationship(carol, "HATES", bob), carol)
try:
_ = path[7]
except IndexError:
assert True
else:
assert False
def test_inequality(self):
other_rel = Relationship(alice, "KNOWS", bob, since=1998)
assert alice_knows_bob != other_rel
def test_cannot_cast_6_tuple(self):
with self.assertRaises(TypeError):
Relationship.cast(("Alice", "KNOWS", "Bob", "foo", "bar", "baz"))
def test_construction_from_more_arguments(self):
with self.assertRaises(TypeError):
Relationship(alice, "KNOWS", bob, carol)
def test_equality_with_similar(self):
other_rel = Relationship(alice, "KNOWS", bob, since=1999)
self.assertEqual(alice_knows_bob, other_rel)
def test_construction_from_three_arguments(self):
rel = Relationship(alice, "KNOWS", bob)
assert rel.start_node is alice
assert rel.end_node is bob
self.assertIs(type(rel), KNOWS)
def test_construction_from_node_and_type_arguments(self):
rel = Relationship(alice, "LIKES")
assert rel.start_node is alice
assert rel.end_node is alice
self.assertEqual(type(rel).__name__, "LIKES")
def test_construction_from_two_node_arguments(self):
rel = Relationship(alice, bob)
assert rel.start_node is alice
assert rel.end_node is bob
self.assertEqual(type(rel).__name__, "Relationship")
def test_construction_from_no_arguments(self):
with self.assertRaises(TypeError):
_ = Relationship()
def test_cannot_cast_relationship_from_generic_object(self):
class Foo(object):
pass
foo = Foo()
with self.assertRaises(ValueError):
Relationship.cast((Node(), foo, Node()))
