def worstcase(r: Redis):
g = Graph('simple_2', r)
v0 = Node(properties={'name': 'v0'})
v1 = Node(properties={'name': 'v1'})
v2 = Node(properties={'name': 'v2'})
v3 = Node(properties={'name': 'v3'})
v4 = Node(properties={'name': 'v4'})
g.add_node(v0)
g.add_node(v1)
g.add_node(v2)
g.add_node(v3)
g.add_node(v4)
e1 = Edge(v0, 'A', v1, properties={'name': 'r1'})
e2 = Edge(v1, 'A', v2, properties={'name': 'r2'})
e3 = Edge(v2, 'A', v0, properties={'name': 'r3'})
e4 = Edge(v0, 'B', v3, properties={'name': 'r5'})
e5 = Edge(v3, 'B', v0, properties={'name': 'r6'})
g.add_edge(e1)
g.add_edge(e2)
g.add_edge(e3)
g.add_edge(e4)
g.add_edge(e5)
g.commit()
return g
def create_simple(r: Redis):
g = Graph('simple', r)
v0 = Node(label='v0', properties={'name': 'v0'})
v1 = Node(label='v1', properties={'name': 'v1'})
v2 = Node(label='v2', properties={'name': 'v2'})
v3 = Node(label='v3', properties={'name': 'v3'})
v4 = Node(label='v4', properties={'name': 'v4'})
g.add_node(v0)
g.add_node(v1)
g.add_node(v2)
g.add_node(v3)
g.add_node(v4)
e1 = Edge(v0, 'r0', v1, properties={'name': 'r0'})
e2 = Edge(v1, 'r1', v2, properties={'name': 'r1'})
e3 = Edge(v2, 'r0', v3, properties={'name': 'r0'})
e4 = Edge(v3, 'r1', v4, properties={'name': 'r1'})
g.add_edge(e1)
g.add_edge(e2)
g.add_edge(e3)
g.add_edge(e4)
g.commit()
return g
def test04_repeated_edges(self):
graphname = "repeated_edges"
g = Graph(graphname, redis_con)
src = Node(label='p', properties={'name': 'src'})
dest = Node(label='p', properties={'name': 'dest'})
edge1 = Edge(src, 'e', dest, properties={'val': 1})
edge2 = Edge(src, 'e', dest, properties={'val': 2})
g.add_node(src)
g.add_node(dest)
g.add_edge(edge1)
g.add_edge(edge2)
g.commit()
# Verify the new edge
q = """MATCH (a)-[e]->(b) RETURN e.val, a.name, b.name ORDER BY e.val"""
actual_result = g.query(q)
expected_result = [[
edge1.properties['val'], src.properties['name'],
dest.properties['name']
],
[
edge2.properties['val'], src.properties['name'],
dest.properties['name']
]]
assert (actual_result.result_set == expected_result)
# Save RDB & Load from RDB
redis_con.execute_command("DEBUG", "RELOAD")
# Verify that the latest edge was properly saved and loaded
actual_result = g.query(q)
assert (actual_result.result_set == expected_result)
def test14_post_deletion_traversal_directions(self):
self.env.flush()
redis_con = self.env.getConnection()
redis_graph = Graph("G", redis_con)
nodes = {}
# Create entities.
labels = ["Dest", "Src", "Src2"]
for idx, l in enumerate(labels):
node = Node(label=l, properties={"val": idx})
redis_graph.add_node(node)
nodes[l] = node
edge = Edge(nodes["Src"], "R", nodes["Dest"])
redis_graph.add_edge(edge)
edge = Edge(nodes["Src2"], "R", nodes["Dest"])
redis_graph.add_edge(edge)
redis_graph.commit()
# Delete a node.
query = """MATCH (n:Src2) DELETE n"""
actual_result = redis_graph.query(query)
self.env.assertEquals(actual_result.nodes_deleted, 1)
self.env.assertEquals(actual_result.relationships_deleted, 1)
query = """MATCH (n1:Src)-[*]->(n2:Dest) RETURN COUNT(*)"""
actual_result = redis_graph.query(query)
expected_result = [[1]]
self.env.assertEquals(actual_result.result_set, expected_result)
# Perform the same traversal, this time traveling from destination to source.
query = """MATCH (n1:Src)-[*]->(n2:Dest {val: 0}) RETURN COUNT(*)"""
actual_result = redis_graph.query(query)
expected_result = [[1]]
self.env.assertEquals(actual_result.result_set, expected_result)
def main():
r = redis.Redis(decode_responses=True)
r.delete('social')
redis_graph = Graph('social', r)
john = Node(label='person', properties={'name':'John Doe', 'age': 33, 'gender':'male', 'status': 'single'})
redis_graph.add_node(john)
japan = Node(label='country', properties={'name':'Japan'})
redis_graph.add_node(japan)
edge_john = Edge(john, 'visited', japan, properties={'purpose':'pleasure'})
redis_graph.add_edge(edge_john)
pearl = Node(label='person', properties={'name':'Pearl White', 'age':25, 'gender':'female', 'status':'married'})
redis_graph.add_node(pearl)
australia = Node(label='country', properties={'name':'Australia'})
redis_graph.add_node(australia)
edge_pearl = Edge(pearl, 'visited', australia, properties={'purpose':'business'})
redis_graph.add_edge(edge_pearl)
mary = Node(label='person', properties={'name':'Mary Mueller', 'age':45, 'gender':'divers', 'status':'divers'})
redis_graph.add_node(mary)
germany = Node(label='country', properties={'name':'Germany'})
redis_graph.add_node(germany)
edge_mary = Edge(mary, 'visited', germany, properties={'purpose':'business'})
redis_graph.add_edge(edge_mary)
redis_graph.commit()
for i in ['pleasure', 'business']:
print('==== Purpose: {} ===='.format(i))
query = '''MATCH (p:person)-[v:visited {{purpose:"{}"}}]->(c:country)
RETURN p.name, p.age, p.status, c.name'''.format(i)
result = redis_graph.query(query)
print_res(result)
def populate_graph(self, graph_name):
# quick return if graph already exists
if redis_con.exists(graph_name):
return redis_graph
people = ["Roi", "Alon", "Ailon", "Boaz", "Tal", "Omri", "Ori"]
visits = [("Roi", "USA"), ("Alon", "Israel"), ("Ailon", "Japan"),
("Boaz", "United Kingdom")]
countries = ["Israel", "USA", "Japan", "United Kingdom"]
redis_graph = Graph(graph_name, redis_con)
personNodes = {}
countryNodes = {}
# create nodes
for p in people:
person = Node(label="person",
properties={
"name": p,
"height": random.randint(160, 200)
})
redis_graph.add_node(person)
personNodes[p] = person
for p in countries:
country = Node(label="country",
properties={
"name": p,
"population": random.randint(100, 400)
})
redis_graph.add_node(country)
countryNodes[p] = country
# create edges
for v in visits:
person = v[0]
country = v[1]
edge = Edge(personNodes[person],
'visit',
countryNodes[country],
properties={'purpose': 'pleasure'})
redis_graph.add_edge(edge)
redis_graph.commit()
# delete nodes, to introduce deleted item within our datablock
query = """MATCH (n:person) WHERE n.name = 'Roi' or n.name = 'Ailon' DELETE n"""
redis_graph.query(query)
query = """MATCH (n:country) WHERE n.name = 'USA' DELETE n"""
redis_graph.query(query)
# create indices
actual_result = redis_con.execute_command(
"GRAPH.QUERY", graph_name, "CREATE INDEX ON :person(name, height)")
actual_result = redis_con.execute_command(
"GRAPH.QUERY", graph_name,
"CREATE INDEX ON :country(name, population)")
return redis_graph
def dump_data(n_records):
r = redis.Redis(host='localhost', port=6379)
data = data_gen(n_records)
redis_graph = Graph('file_activity1', r)
nodes = {}
edges = {}
pprint(data)
for rec in data:
_node = Node(label='file',
properties={
'fid': rec['_id'],
'name': rec['name'],
'date_added': rec['date_added'],
'platform': rec['platform']
})
r.set(rec['_id'], _node.alias)
redis_graph.add_node(_node)
nodes[rec['_id']] = _node
for rec in data:
for fileid, time_stamp in rec['downloaded']:
edge = Edge(nodes[rec['_id']],
'DOWNLOADED',
nodes[fileid],
properties={
'time': time_stamp,
'activity': 'downloaded'
})
redis_graph.add_edge(edge)
for fileid, time_stamp in rec['executed']:
edge = Edge(nodes[rec['_id']],
'EXECUTED',
nodes[fileid],
properties={
'time': time_stamp,
'activity': 'executed'
})
redis_graph.add_edge(edge)
for fileid, time_stamp in rec['removed']:
edge = Edge(nodes[rec['_id']],
'REMOVED',
nodes[fileid],
properties={
'time': time_stamp,
'activity': 'removed'
})
redis_graph.add_edge(edge)
redis_graph.commit()
print("Graph created")
class InitialGraph:
def get_apistructure(self, entrypoint_node, api_doc):
""" It breaks the endpoint into two parts collection and classes"""
self.collection_endpoints = {}
self.class_endpoints = {}
print(
"split entrypoint into 2 types of endpoints collection and classes"
)
for support_property in api_doc.entrypoint.entrypoint.supportedProperty:
if isinstance(support_property,
hydrus.hydraspec.doc_writer.EntryPointClass):
self.class_endpoints[
support_property.name] = support_property.id_
if isinstance(support_property,
hydrus.hydraspec.doc_writer.EntryPointCollection):
self.collection_endpoints[
support_property.name] = support_property.id_
if len(self.class_endpoints.keys()) > 0:
clas = ClassEndpoints(self.redis_graph, self.class_endpoints)
clas.endpointclasses(entrypoint_node, api_doc, self.url)
if len(self.collection_endpoints.keys()) > 0:
coll = CollectionEndpoints(self.redis_graph, self.class_endpoints)
coll.endpointCollection(self.collection_endpoints, entrypoint_node,
api_doc, self.url)
def get_endpoints(self, api_doc, redis_connection):
"""Create node for entrypoint"""
print("creating entrypoint node")
entrypoint_properties = {}
entrypoint_properties["@id"] = str("vocab:Entrypoint")
entrypoint_properties["url"] = str(api_doc.entrypoint.url) + str(
api_doc.entrypoint.api)
entrypoint_properties["supportedOperation"] = "GET"
entrypoint_node = Node(label="id",
alias="Entrypoint",
properties=entrypoint_properties)
self.redis_graph.add_node(entrypoint_node)
redis_connection.set("EntryPoint", entrypoint_properties)
return self.get_apistructure(entrypoint_node, api_doc)
def main(self, new_url, api_doc, check_commit):
redis_connection = RedisProxy()
redis_con = redis_connection.get_connection()
self.url = new_url
self.redis_graph = Graph("apidoc", redis_con)
print("loading... of graph")
self.get_endpoints(api_doc, redis_con)
if check_commit:
print("commiting")
self.redis_graph.commit()
# creating whole the graph in redis
print("done!!!!")
def setUpClass(cls):
print "QueryValidationFlowTest"
global redis_graph
cls.r = disposable_redis()
cls.r.start()
redis_con = cls.r.client()
# Create a single graph.
redis_graph = Graph("G", redis_con)
node = Node(properties={"age": 34})
redis_graph.add_node(node)
redis_graph.commit()
def empty_graph():
global redis_graph
redis_con = _brand_new_redis()
redis_graph = Graph("G", redis_con)
# Create a graph with a single node.
redis_graph.add_node(Node())
redis_graph.commit()
# Delete node to have an empty graph.
redis_graph.query("MATCH (n) DELETE n")
def populate_graph(cls):
global redis_graph
redis_con = cls.r.client()
redis_graph = Graph(GRAPH_ID, redis_con)
# Create entities
for i in range(10):
node = Node(label="person", properties={"id": i})
redis_graph.add_node(node)
redis_graph.commit()
# Make sure node id attribute matches node's internal ID.
query = """MATCH (n) SET n.id = ID(n)"""
redis_graph.query(query)
def test_stringify_query_result(self):
redis_graph = Graph('stringify', self.r)
john = Node(alias='a',
label='person',
properties={
'name': 'John Doe',
'age': 33,
'gender': 'male',
'status': 'single'
})
redis_graph.add_node(john)
japan = Node(alias='b', label='country', properties={'name': 'Japan'})
redis_graph.add_node(japan)
edge = Edge(john, 'visited', japan, properties={'purpose': 'pleasure'})
redis_graph.add_edge(edge)
self.assertEqual(
str(john),
"""(a:person{age:33,gender:"male",name:"John Doe",status:"single"})"""
)
self.assertEqual(
str(edge),
"""(a:person{age:33,gender:"male",name:"John Doe",status:"single"})"""
+ """-[:visited{purpose:"pleasure"}]->""" +
"""(b:country{name:"Japan"})""")
self.assertEqual(str(japan), """(b:country{name:"Japan"})""")
redis_graph.commit()
query = """MATCH (p:person)-[v:visited {purpose:"pleasure"}]->(c:country)
RETURN p, v, c"""
result = redis_graph.query(query)
person = result.result_set[0][0]
visit = result.result_set[0][1]
country = result.result_set[0][2]
self.assertEqual(
str(person),
"""(:person{age:33,gender:"male",name:"John Doe",status:"single"})"""
)
self.assertEqual(str(visit),
"""()-[:visited{purpose:"pleasure"}]->()""")
self.assertEqual(str(country), """(:country{name:"Japan"})""")
redis_graph.delete()
def create_friends(r: Redis):
g = Graph('friends', r)
adam = Node(label='User', properties={'name': 'Adam', 'mail': 'f****r'})
pernilla = Node(label='User', properties={'name': 'Pernilla'})
david = Node(label='User', properties={'name': 'David'})
g.add_node(adam)
g.add_node(pernilla)
g.add_node(david)
g.add_edge(Edge(adam, 'FRIEND', pernilla))
g.add_edge(Edge(pernilla, 'FRIEND', david))
g.commit()
return g
def populate_acyclic_graph(self):
global acyclic_graph
acyclic_graph = Graph("G", redis_con)
# Construct a graph with the form:
# (v1)-[:E]->(v2)-[:E]->(v3)
node_props = ['v1', 'v2', 'v3']
nodes = []
for idx, v in enumerate(node_props):
node = Node(label="L", properties={"val": v})
nodes.append(node)
acyclic_graph.add_node(node)
edge = Edge(nodes[0], "E", nodes[1])
acyclic_graph.add_edge(edge)
edge = Edge(nodes[1], "E", nodes[2])
acyclic_graph.add_edge(edge)
acyclic_graph.commit()
def test_graph_creation(self):
redis_graph = Graph('social', self.r)
john = Node(label='person',
properties={
'name': 'John Doe',
'age': 33,
'gender': 'male',
'status': 'single'
})
redis_graph.add_node(john)
japan = Node(label='country', properties={'name': 'Japan'})
redis_graph.add_node(japan)
edge = Edge(john, 'visited', japan, properties={'purpose': 'pleasure'})
redis_graph.add_edge(edge)
redis_graph.commit()
query = (
'MATCH (p:person)-[v:visited {purpose:"pleasure"}]->(c:country) '
'RETURN p, v, c')
result = redis_graph.query(query)
person = result.result_set[0][0]
visit = result.result_set[0][1]
country = result.result_set[0][2]
self.assertEqual(person, john)
self.assertEqual(visit.properties, edge.properties)
self.assertEqual(country, japan)
query = """RETURN [1, 2.3, "4", true, false, null]"""
result = redis_graph.query(query)
self.assertEqual([1, 2.3, "4", True, False, None],
result.result_set[0][0])
# All done, remove graph.
redis_graph.delete()
def test():
r = redis.Redis(host='localhost', port=6379)
redis_graph = Graph('social', r)
john = Node(label='person',
properties={
'name': 'John Doe',
'age': 33,
'gender': 'male',
'status': 'single'
})
redis_graph.add_node(john)
japan = Node(label='country', properties={'name': 'Japan'})
redis_graph.add_node(japan)
edge = Edge(john, 'visited', japan, properties={'purpose': 'pleasure'})
redis_graph.add_edge(edge)
redis_graph.commit()
query = """MATCH (p:person)-[v:visited {purpose:"pleasure"}]->(c:country)
RETURN p.name, p.age, v.purpose, c.name"""
result = redis_graph.query(query)
# Print resultset
result.pretty_print()
# Iterate through resultset, skip header row at position 0
for record in result.result_set[1:]:
person_name = record[0]
person_age = record[1]
visit_purpose = record[2]
country_name = record[3]
def populate_cyclic_graph(self):
global graph_with_cycle
graph_with_cycle = Graph("H", redis_con)
# Construct a graph with the form:
# (v1)-[:E]->(v2)-[:E]->(v3), (v2)-[:E]->(v1)
node_props = ['v1', 'v2', 'v3']
nodes = []
for idx, v in enumerate(node_props):
node = Node(label="L", properties={"val": v})
nodes.append(node)
graph_with_cycle.add_node(node)
edge = Edge(nodes[0], "E", nodes[1])
graph_with_cycle.add_edge(edge)
edge = Edge(nodes[1], "E", nodes[2])
graph_with_cycle.add_edge(edge)
# Introduce a cycle between v2 and v1.
edge = Edge(nodes[1], "E", nodes[0])
graph_with_cycle.add_edge(edge)
graph_with_cycle.commit()
class CSVImporter:
def __init__(self, args):
self.host = args.host
self.port = args.port
self.graph = args.graph
self.file = open(args.file, 'r')
self.object = args.object
self.delimiter = args.delimiter
self.rows = args.rows
self.hasHeader = args.header
self.redis = redis.Redis(args.host, args.port)
self.client = Graph(self.graph, self.redis)
def addNode(self, row):
args = {}
idx = 0
if len(row) < 2 or row[1] != 'airport':
return
for val in row:
idx += 1
if idx == 1:
args['id'] = int(val)
elif idx == 4:
args['code'] = val
elif idx == 5:
args['icao'] = val
elif idx == 6:
args['description'] = val
elif idx == 11:
args['country'] = val
elif idx == 12:
args['city'] = val
print(args)
node = Node(label='airport', properties=args)
self.client.add_node(node)
def addEdge(self, row):
if len(row) < 4 or row[3] != 'route':
return
idx = 0
from_id = ''
to_id = ''
dist = ''
for val in row:
idx += 1
if idx == 2:
from_id = val
elif idx == 3:
to_id = val
elif idx == 5:
dist = val
query = 'MATCH (from:airport { id: ' + from_id + ' }), (to:airport { id: ' + to_id + ' }) CREATE (from)-[:ROUTE {dist: ' + dist + ' }]->(to)'
result = self.client.query(query)
def loafFile(self):
reader = csv.reader(self.file, delimiter=self.delimiter)
if self.hasHeader == True:
next(reader)
n = 0
for row in reader:
if self.rows > 0 and n == self.rows:
break
if self.object == 'nodes':
self.addNode(row)
elif self.object == 'edges':
self.addEdge(row)
n += 1
if self.object == 'nodes':
self.client.commit()
print('Finished loading ' + str(n) + ' rows.')
class rgraph(absgraph):
def __init__(self, server, schema):
host, port = server.split(':')
self._client_stub = redis.Redis(host=host, port=int(port))
self._graph = Graph('test', self._client_stub)
self._maxquery = 100
self.cypher_no_exists = True
super(rgraph, self).__init__(schema)
def drop_all(self):
self._graph.delete()
def get_schema(self):
pass
def load_schema(self, schema):
pass
def set_index(self):
for p, d in self.schema.items():
if not 'index' in d:
continue
elif not d['index']:
continue
query = 'CREATE INDEX ON :node (%s)' % p
self.query(query)
def _is_intinstance(self, p, v):
return self._int_type(p) and (isinstance(v, np.float_) or isinstance(
v, float) or isinstance(v, str))
def add_node(self, node):
self._graph.add_node(node)
def add_edge(self, edge):
self._graph.add_edge(edge)
def add_nodes(self, nodes):
for i in nodes:
node = Node(label='node', properties=i)
self.add_node(node)
def nquads(self, df, predicates, i):
#d = self.id_predicate
#nodes = [Node(label='node', properties={'%s'%d: df.iloc[i]['%s'%d],
# 'numeric': srlz(df.iloc[i]['numeric'])}) for i in range(n, n+m)]
plen = len(predicates)
lim = min(i + self._maxquery, len(df))
nquads = []
while i < lim:
properties = {}
for p in predicates:
try:
s = df.iloc[i][p]
except:
s = i
s = self.serialize(s)
properties[p] = s
nquads.append(properties)
i += 1
return nquads, i
def nquads_edges(self, graph, label, i=0, nodes=None, neighbors=None, j=0):
if nodes is None:
nodes = list(graph.nodes())
edges = []
budget = self._maxquery
for node in nodes[i:]:
if neighbors is None:
neighbors = list(graph.neighbors(node))
for k, neigh in enumerate(neighbors[j:]):
if budget == 0:
return edges, i, neighbors, j + k, self._maxquery
edges.append((node, neigh))
budget -= 1
i += 1
neighbors = None
j = 0
return edges, i, neighbors, j + k, self._maxquery - budget
def nquads_edges2(self, edges):
"""
GRAPH.QUERY test 'MATCH (a:node {name: "acc-tight5.mps.pkl__v998"}),
(b:node {name: "acc-tight5.mps.pkl__v998"}), (c:node {name:
"acc-tight5.mps.pkl__v10"}), (d:node {name:
"acc-tight5.mps.pkl__slack-min"}) CREATE (a)-[:edge]->(b),
(c)-[:edge]->(d)'
"""
# get all the nodes
if isinstance(edges[0][0], str):
base = '(s%d:node {%s: "%s"}), (d%d:node {%s: "%s"})'
else:
base = '(s%d:node {%s: %s}), (d%d:node {%s: %s})'
l = []
#q = 'MATCH '
#for i, e in enumerate(edges):
# if i > 0:
# q += ', '
# q += base%(i, self.id_predicate, e[0], i, self.id_predicate, e[1])
# l.append('(s%d)-[:%s]->(d%d)'%(i, self.edge_attribute, i))
#q += ' CREATE ' + ', '.join(l)
m = []
for i, e in enumerate(edges):
m.append(base %
(i, self.id_predicate, e[0], i, self.id_predicate, e[1]))
l.append('(s%d)-[:%s]->(d%d)' % (i, self.edge_attribute, i))
q = 'MATCH ' + ', '.join(m) + ' CREATE ' + ', '.join(l)
return q
def parse_neighbors(self, res, ret):
if len(res.result_set) == 0:
return
p = res.result_set[0][0].decode().split('.')[1] # XXX
for k, v in res.result_set[1:]:
k = k.decode()
v = v.decode()
if self._is_intinstance(p, k):
k = int(float(k))
if self._is_intinstance(p, v):
v = int(float(v))
if k in ret:
ret[k].append(v)
else:
ret[k] = [v]
def neighbors(self, identities, pred=None, id_pred=None):
"""
We want something like
GRAPH.QUERY test
'MATCH (n: node {name:"acc-tight5.mps.pkl__v998"})-[:edge]->(m)
RETURN m.numeric, m.name'
or better solution like:
GRAPH.QUERY test
'MATCH (n0: node)-[:edge]->(m)
WHERE n0.name = "acc-tight5.mps.pkl__v998"
OR n0.name = "acc-tight5.mps.pkl__v10"
RETURN n0.name, m.name'
"""
if not id_pred:
id_pred = self.id_predicate
if not pred:
pred = self.edge_attribute
ret = OrderedDict()
#
# For small numbers, this is faster
#
if len(identities) < 1:
for i in identities:
q = 'MATCH (s: node {%s: "%s"})-[:%s]->(d) RETURN d.%s' % (
id_pred, i, pred, id_pred)
res = self.query(q)
ret.update({i: [j[0].decode() for j in res.result_set[1:]]})
return ret
#
# Otherwise batch requests
#
if isinstance(identities[0], str):
where_base = 'n.%s = "%s"'
else:
where_base = 'n.%s = %s'
step = 8
l = 0
h = step
lim = len(identities)
ret = OrderedDict()
while l < lim:
b = identities[l:h]
#where = ['n.%s = "%s"'%(id_pred, i) for i in b]
where = [where_base % (id_pred, i) for i in b]
where = ' OR '.join(where)
query = ('MATCH (n: node)-[:%s]->(m) WHERE %s RETURN n.%s, m.%s' %
(pred, where, id_pred, id_pred))
res = self.query(query)
self.parse_neighbors(res, ret)
l = h
h = h + step
return ret
def _int_type(self, predicate):
return self.schema[predicate]['type'] == 'int'
def parse_batch(self, res, ret, predicates):
r = res.result_set[1]
npreds = len(predicates)
for i, p in enumerate(predicates):
l = [j.decode() for j in r[i::npreds]]
t = self.deserialize_type(p)
if t:
l = self.deserialize(l, t)
# XXX
if self._int_type(p) and (isinstance(l[0], np.float_)
or isinstance(l[0], float)
or isinstance(l[0], str)):
l = [int(float(i)) for i in l]
#if p in ret:
# ret[p].extend(l)
#else:
# ret[p] = l
ret[p].extend(l)
def batch(self, identities, predicates, identities_predicate=None):
if identities_predicate is None:
identities_predicate = self.id_predicate
if isinstance(identities[0], str):
mtch_base = '(n%d:node {%s: "%s"})'
else:
mtch_base = '(n%d:node {%s: %s})'
step = 1000
l = 0
h = step
lim = len(identities)
ret = OrderedDict({p: [] for p in predicates})
while l < lim:
nodes = identities[l:h]
mtch = [
mtch_base % (j, identities_predicate, i)
for j, i in enumerate(nodes)
]
mtch = ', '.join(mtch)
rtrn = [
'n%d.%s' % (j, p) for j in range(len(nodes))
for p in predicates
]
rtrn = ', '.join(rtrn)
query = 'MATCH ' + mtch + ' RETURN ' + rtrn
res = self.query(query)
# We get something like
# [[b'n0.numeric', b'n0.name', b'n1.numeric', b'n1.name'], ...
# res.result_set[1][0] is like b'xazf'
self.parse_batch(res, ret, predicates)
l = h
h = h + step
return ret
def missing_values(self, predicate, low, high):
return []
def _one_cypher(self, predicate, identity):
#query = 'MATCH (n:node {%s: "%s"}) RETURN n.%s'%(
# self.sorted_predicate, identity, predicate)
if identity:
if isinstance(identity, str):
query = 'MATCH (n:node) WHERE n.%s = "%s" RETURN n.%s' % (
self.id_predicate, identity, predicate)
else:
query = 'MATCH (n:node) WHERE n.%s = "%s" RETURN n.%s' % (
self.id_predicate, identity, predicate)
else:
if self.cypher_no_exists:
whr = 'n.%s != ""'
else:
whr = 'exists(n.%s)'
query = ('MATCH (n:node) WHERE ' + whr +
' RETURN n.%s LIMIT 1') % (predicate, predicate)
return query
def parse_one(self, res, predicate):
return res.result_set[1][0].decode()
def one(self, predicate, identity=None):
query = self._one_cypher(predicate, identity)
res = self.query(query)
r = self.parse_one(res, predicate)
t = self.deserialize_type(predicate)
if t:
r = self.deserialize([r], t)[0]
return r
def _count_cypher(self, name=None):
if name is None:
name = self.id_predicate
return 'MATCH (n:node) RETURN COUNT(n)'
def parse_count(self, res):
return (int(float(res.result_set[1][0].decode())))
def count(self, name=None):
query = self._count_cypher(name)
res = self.query(query)
return self.parse_count(res)
def merge(self):
query = ''
for _, node in self._graph.nodes.items():
query += str(node) + ','
for edge in self._graph.edges:
query += str(edge) + ','
# Discard leading comma.
if query[-1] is ',':
query = query[:-1]
self._graph.merge(query)
def commit(self):
self._graph.commit()
def flush(self):
self._graph.flush()
def query(self, query):
return self._graph.query(query)
def range_cypher(self, low, high, predicates, id_predicate, expand):
unsortable = False
if id_predicate is None:
id_predicate = self.id_predicate
if self.id_predicate_unsortable:
unsortable = True
rtrn = ['n.%s' % p for p in predicates]
rtrn = ', '.join(rtrn)
if unsortable:
query = ('MATCH (n: node) RETURN %s ORDER BY n.%s LIMIT %d' %
(rtrn, self.sorted_predicate, high - low))
else:
pred = self.sorted_predicate
query = ('MATCH (n: node) WHERE n.%s >= %d AND n.%s < %d '
'RETURN %s ORDER BY n.%s' %
(pred, low, pred, high, rtrn, pred))
return query
def _range_xform(self, ret, predicates):
ret2 = [{} for i in range(len(ret[predicates[0]]))]
for k, vs in ret.items():
for j, v in enumerate(vs):
ret2[j][k] = v
return ret2
def _range(self, low, high, predicates, id_predicate=None, expand=False):
query = self.range_cypher(low, high, predicates, id_predicate, expand)
ret = OrderedDict()
res = self.query(query)
res_predicates = [p.decode().split('.')[1] for p in res.result_set[0]]
for pi, p in enumerate(predicates):
tmp = [i[pi].decode() for i in res.result_set[1:]]
ret[p] = tmp
# for RETURN n.numeric, n.name, n.identity, ret[1:] is like
# [[b'numericvalue0', b'namevalue0', b'identityvalue0'], ..]
# Now ret looks like [{'name': [name values]}, {'numeric': []} ..]
# To be compatible with _dataframe, we transform this to
# [{'name': 'namevalue0', 'numeric': 'numericvalue0'}, {}, {} ... ]
ret2 = self._range_xform(ret, predicates)
return ret2
def load_df(self, df, predicates, n=0):
print('loading nodes')
while n < len(df):
nquads, n = self.nquads(df, predicates, n)
self.add_nodes(nquads)
self.flush()
def load_graph(self, g, edge):
nodes = list(g.nodes())
print('loading edges')
n = 0
j = 0
nbrs = None
num_nodes = len(nodes)
n_prev = 0
while n < num_nodes:
nquads, n, nbrs, j, c = self.nquads_edges(g,
edge,
n,
nodes=nodes,
neighbors=nbrs,
j=j)
query = self.nquads_edges2(nquads)
self.query(query)
if n > n_prev + 10000:
print('%d / %d' % (n, num_nodes))
n_prev = n
class RedisGraph:
""" Graph abstraction over RedisGraph. A thin wrapper but provides us some options. """
def __init__(self,
host='localhost',
port=6379,
graph='default',
password=''):
""" Construct a connection to Redis Graph. """
self.r = redis.Redis(host=host, port=port, password=password)
self.redis_graph = Graph(graph, self.r)
def add_node(self, identifier=None, label=None, properties=None):
""" Add a node with the given label and properties. """
logger.debug(
f"--adding node id:{identifier} label:{label} prop:{properties}")
if identifier and properties:
properties['id'] = identifier
node = Node(node_id=identifier,
alias=identifier,
label=label,
properties=properties)
self.redis_graph.add_node(node)
return node
def get_edge(self, start, end, predicate=None):
""" Get an edge from the graph with the specified start and end identifiers. """
result = None
for edge in self.redis_graph.edges:
if edge.src_node.id == start and edge.dest_node.id == end:
result = edge
break
return result
def add_edge(self, start, predicate, end, properties={}):
""" Add an edge with the given predicate and properties between start and end nodes. """
logger.debug(
f"--adding edge start:{start} pred:{predicate} end:{end} prop:{properties}"
)
if isinstance(start, str) and isinstance(end, str):
start = Node(node_id=start, label='thing')
end = Node(node_id=end, label='thing')
self.redis_graph.add_node(start)
self.redis_graph.add_node(end)
edge = Edge(start, predicate, end, properties)
self.redis_graph.add_edge(edge)
return edge
def has_node(self, identifier):
return identifier in self.redis_graph.nodes
def get_node(self, identifier, properties=None):
return self.redis_graph.nodes[identifier]
def commit(self):
""" Commit modifications to the graph. """
self.redis_graph.commit()
def query(self, query):
""" Query and return result set. """
result = self.redis_graph.query(query)
result.pretty_print()
return result
def delete(self):
""" Delete the named graph. """
self.redis_graph.delete()
class testConcurrentQueryFlow(FlowTestsBase):
def __init__(self):
self.env = Env(decodeResponses=True)
# skip test if we're running under Valgrind
if self.env.envRunner.debugger is not None:
self.env.skip(
) # valgrind is not working correctly with multi processing
self.conn = self.env.getConnection()
self.graph = Graph(GRAPH_ID, self.conn)
self.populate_graph()
def populate_graph(self):
nodes = {}
# Create entities
for p in people:
node = Node(label="person", properties={"name": p})
self.graph.add_node(node)
nodes[p] = node
# Fully connected graph
for src in nodes:
for dest in nodes:
if src != dest:
edge = Edge(nodes[src], "know", nodes[dest])
self.graph.add_edge(edge)
self.graph.commit()
# Count number of nodes in the graph
def test01_concurrent_aggregation(self):
q = """MATCH (p:person) RETURN count(p)"""
queries = [q] * CLIENT_COUNT
results = run_concurrent(queries, thread_run_query)
for result in results:
person_count = result["result_set"][0][0]
self.env.assertEqual(person_count, len(people))
# Concurrently get neighbors of every node.
def test02_retrieve_neighbors(self):
q = """MATCH (p:person)-[know]->(n:person) RETURN n.name"""
queries = [q] * CLIENT_COUNT
results = run_concurrent(queries, thread_run_query)
# Fully connected graph + header row.
expected_resultset_size = len(people) * (len(people) - 1)
for result in results:
self.env.assertEqual(len(result["result_set"]),
expected_resultset_size)
# Concurrent writes
def test_03_concurrent_write(self):
queries = [
"""CREATE (c:country {id:"%d"})""" % i for i in range(CLIENT_COUNT)
]
results = run_concurrent(queries, thread_run_query)
for result in results:
self.env.assertEqual(result["nodes_created"], 1)
self.env.assertEqual(result["properties_set"], 1)
# Try to delete graph multiple times.
def test_04_concurrent_delete(self):
pool = Pool(nodes=CLIENT_COUNT)
# invoke queries
assertions = pool.map(delete_graph, [GRAPH_ID] * CLIENT_COUNT)
# Exactly one thread should have successfully deleted the graph.
self.env.assertEquals(assertions.count(True), 1)
# Try to delete a graph while multiple queries are executing.
def test_05_concurrent_read_delete(self):
##############################################################################################
# Delete graph via Redis DEL key.
##############################################################################################
self.populate_graph()
pool = Pool(nodes=CLIENT_COUNT)
manager = pathos_multiprocess.Manager()
barrier = manager.Barrier(CLIENT_COUNT)
barriers = [barrier] * CLIENT_COUNT
q = """UNWIND (range(0, 10000)) AS x WITH x AS x WHERE (x / 900) = 1 RETURN x"""
queries = [q] * CLIENT_COUNT
# invoke queries
m = pool.amap(thread_run_query, queries, barriers)
self.conn.delete(GRAPH_ID)
# wait for processes to return
m.wait()
# get the results
results = m.get()
# validate result.
self.env.assertTrue(
all([r["result_set"][0][0] == 900 for r in results]))
# Make sure Graph is empty, e.g. graph was deleted.
resultset = self.graph.query("MATCH (n) RETURN count(n)").result_set
self.env.assertEquals(resultset[0][0], 0)
##############################################################################################
# Delete graph via Redis FLUSHALL.
##############################################################################################
self.populate_graph()
q = """UNWIND (range(0, 10000)) AS x WITH x AS x WHERE (x / 900) = 1 RETURN x"""
queries = [q] * CLIENT_COUNT
barrier = manager.Barrier(CLIENT_COUNT)
barriers = [barrier] * CLIENT_COUNT
# invoke queries
m = pool.amap(thread_run_query, queries, barriers)
self.conn.flushall()
# wait for processes to return
m.wait()
# get the results
results = m.get()
# validate result.
self.env.assertTrue(
all([r["result_set"][0][0] == 900 for r in results]))
# Make sure Graph is empty, e.g. graph was deleted.
resultset = self.graph.query("MATCH (n) RETURN count(n)").result_set
self.env.assertEquals(resultset[0][0], 0)
##############################################################################################
# Delete graph via GRAPH.DELETE.
##############################################################################################
self.populate_graph()
q = """UNWIND (range(0, 10000)) AS x WITH x AS x WHERE (x / 900) = 1 RETURN x"""
queries = [q] * CLIENT_COUNT
barrier = manager.Barrier(CLIENT_COUNT)
barriers = [barrier] * CLIENT_COUNT
# invoke queries
m = pool.amap(thread_run_query, queries, barriers)
self.graph.delete()
# wait for processes to return
m.wait()
# get the results
results = m.get()
# validate result.
self.env.assertTrue(
all([r["result_set"][0][0] == 900 for r in results]))
# Make sure Graph is empty, e.g. graph was deleted.
resultset = self.graph.query("MATCH (n) RETURN count(n)").result_set
self.env.assertEquals(resultset[0][0], 0)
def test_06_concurrent_write_delete(self):
# Test setup - validate that graph exists and possible results are None
self.graph.query("MATCH (n) RETURN n")
pool = Pool(nodes=1)
heavy_write_query = """UNWIND(range(0,999999)) as x CREATE(n) RETURN count(n)"""
writer = pool.apipe(thread_run_query, heavy_write_query, None)
self.conn.delete(GRAPH_ID)
writer.wait()
possible_exceptions = [
"Encountered different graph value when opened key " + GRAPH_ID,
"Encountered an empty key when opened key " + GRAPH_ID
]
result = writer.get()
if isinstance(result, str):
self.env.assertContains(result, possible_exceptions)
else:
self.env.assertEquals(1000000, result["result_set"][0][0])
def test_07_concurrent_write_rename(self):
# Test setup - validate that graph exists and possible results are None
self.graph.query("MATCH (n) RETURN n")
pool = Pool(nodes=1)
new_graph = GRAPH_ID + "2"
# Create new empty graph with id GRAPH_ID + "2"
self.conn.execute_command("GRAPH.QUERY", new_graph,
"""MATCH (n) return n""", "--compact")
heavy_write_query = """UNWIND(range(0,999999)) as x CREATE(n) RETURN count(n)"""
writer = pool.apipe(thread_run_query, heavy_write_query, None)
self.conn.rename(GRAPH_ID, new_graph)
writer.wait()
# Possible scenarios:
# 1. Rename is done before query is sent. The name in the graph context is new_graph, so when upon commit, when trying to open new_graph key, it will encounter an empty key since new_graph is not a valid key.
# Note: As from https://github.com/RedisGraph/RedisGraph/pull/820 this may not be valid since the rename event handler might actually rename the graph key, before the query execution.
# 2. Rename is done during query executing, so when commiting and comparing stored graph context name (GRAPH_ID) to the retrived value graph context name (new_graph), the identifiers are not the same, since new_graph value is now stored at GRAPH_ID value.
possible_exceptions = [
"Encountered different graph value when opened key " + GRAPH_ID,
"Encountered an empty key when opened key " + new_graph
]
result = writer.get()
if isinstance(result, str):
self.env.assertContains(result, possible_exceptions)
else:
self.env.assertEquals(1000000, result["result_set"][0][0])
def test_08_concurrent_write_replace(self):
# Test setup - validate that graph exists and possible results are None
self.graph.query("MATCH (n) RETURN n")
pool = Pool(nodes=1)
heavy_write_query = """UNWIND(range(0,999999)) as x CREATE(n) RETURN count(n)"""
writer = pool.apipe(thread_run_query, heavy_write_query, None)
set_result = self.conn.set(GRAPH_ID, "1")
writer.wait()
possible_exceptions = [
"Encountered a non-graph value type when opened key " + GRAPH_ID,
"WRONGTYPE Operation against a key holding the wrong kind of value"
]
result = writer.get()
if isinstance(result, str):
# If the SET command attempted to execute while the CREATE query was running,
# an exception should have been issued.
self.env.assertContains(result, possible_exceptions)
else:
# Otherwise, both the CREATE query and the SET command should have succeeded.
self.env.assertEquals(1000000, result.result_set[0][0])
self.env.assertEquals(set_result, True)
# Delete the key
self.conn.delete(GRAPH_ID)
def test_09_concurrent_multiple_readers_after_big_write(self):
# Test issue #890
self.graph = Graph(GRAPH_ID, self.conn)
self.graph.query("""UNWIND(range(0,999)) as x CREATE()-[:R]->()""")
read_query = """MATCH (n)-[r:R]->(m) RETURN count(r) AS res UNION RETURN 0 AS res"""
queries = [read_query] * CLIENT_COUNT
results = run_concurrent(queries, thread_run_query)
for result in results:
if isinstance(result, str):
self.env.assertEquals(0, result)
else:
self.env.assertEquals(1000, result["result_set"][0][0])
def populate_graph(self, graph_name):
redis_graph = Graph(graph_name, redis_con)
# quick return if graph already exists
if redis_con.exists(graph_name):
return redis_graph
people = ["Roi", "Alon", "Ailon", "Boaz", "Tal", "Omri", "Ori"]
visits = [("Roi", "USA"), ("Alon", "Israel"), ("Ailon", "Japan"),
("Boaz", "United Kingdom")]
countries = ["Israel", "USA", "Japan", "United Kingdom"]
personNodes = {}
countryNodes = {}
# create nodes
for p in people:
person = Node(label="person",
properties={
"name": p,
"height": random.randint(160, 200)
})
redis_graph.add_node(person)
personNodes[p] = person
for c in countries:
country = Node(label="country",
properties={
"name": c,
"population": random.randint(100, 400)
})
redis_graph.add_node(country)
countryNodes[c] = country
# create edges
for v in visits:
person = v[0]
country = v[1]
edge = Edge(personNodes[person],
'visit',
countryNodes[country],
properties={'purpose': 'pleasure'})
redis_graph.add_edge(edge)
redis_graph.commit()
# delete nodes, to introduce deleted entries within our datablock
query = """MATCH (n:person) WHERE n.name = 'Roi' or n.name = 'Ailon' DELETE n"""
redis_graph.query(query)
query = """MATCH (n:country) WHERE n.name = 'USA' DELETE n"""
redis_graph.query(query)
# create indices
redis_con.execute_command(
"GRAPH.QUERY", graph_name,
"CREATE INDEX FOR (p:Person) ON (p.name, p.height)")
redis_con.execute_command(
"GRAPH.QUERY", graph_name,
"CREATE INDEX FOR (c:country) ON (c.name, c.population)")
actual_result = redis_con.execute_command(
"GRAPH.QUERY", graph_name,
"CREATE INDEX FOR ()-[r:visit]-() ON (r.purpose)")
actual_result = redis_con.execute_command(
"GRAPH.QUERY", graph_name,
"CALL db.idx.fulltext.createNodeIndex({label: 'person', stopwords: ['A', 'B'], language: 'english'}, 'text')"
)
return redis_graph
source_entity_id,
destination_entity_id),
'rank', 1)
source_node = Node(label='entity',
properties={
'id': source_entity_id,
'rank': 1,
'name': source_canonical_name
})
destination_node = Node(label='entity',
properties={
'id': destination_entity_id,
'rank': 1,
'name':
destination_canonical_name
})
redis_graph.add_node(source_node)
redis_graph.add_node(destination_node)
edge = Edge(source_node,
'related',
destination_node,
properties={'article': sentence_key})
redis_graph.add_edge(edge)
redis_graph.commit()
if num_sents % 100 == 0:
log(f"... {num_sents} sentences ")
num_sents += 1
log("Flushing graph for sentence %s " % sentence_key)
logger.info("Completed")
