def main_test():
client = None
try:
space_name = 'space_' + threading.current_thread().getName()
print('thread name: %s, space_name : %s' %
(threading.current_thread().getName(), space_name))
# Get one client
client = GraphClient(connection_pool)
auth_resp = client.authenticate('user', 'password')
if auth_resp.error_code:
raise AuthException("Auth failed")
query_resp = client.execute_query('SHOW SPACES')
if has_space(query_resp.rows, space_name):
print('has %s, drop it' % space_name)
do_simple_execute(client, 'DROP SPACE %s' % space_name)
# Create space mySpace
do_simple_execute(client, 'CREATE SPACE %s'
% space_name)
do_simple_execute(client, 'USE %s' % space_name)
time.sleep(1)
# Create tag and edge
do_simple_execute(client, 'CREATE TAG person(name string, age int); '
'CREATE EDGE like(likeness double)')
# It should large than the cycle of loading the schema
time.sleep(6)
# Insert vertex and edge
do_simple_execute(client, 'INSERT VERTEX person(name, age) VALUES 1:(\'Bob\', 10)')
do_simple_execute(client, 'INSERT VERTEX person(name, age) VALUES 2:(\'Lily\', 9)')
do_simple_execute(client, 'INSERT VERTEX person(name, age) VALUES 3:(\'Tom\', 10)')
do_simple_execute(client, 'INSERT VERTEX person(name, age) VALUES 4:(\'Jerry\', 13);INSERT VERTEX person(name, age) VALUES 5:(\'John\', 11)')
do_simple_execute(client, 'INSERT EDGE like(likeness) VALUES 1->2:(80.0)')
do_simple_execute(client, 'INSERT EDGE like(likeness) VALUES 1->3:(70.0)')
do_simple_execute(client, 'INSERT EDGE like(likeness) VALUES 2->4:(84.0), 3->5:(68.3), 1->5:(97.2)')
# Query data
query_resp = client.execute_query('GO FROM 1 OVER like YIELD $$.person.name, '
'$$.person.age, like.likeness')
if query_resp.error_code:
print('Execute failed: %s' % query_resp.error_msg)
exit(1)
# Print the result of query
print(' \n====== The query result of thread[%s]======\n '
% threading.current_thread().getName())
print_value(query_resp.column_names, query_resp.rows)
client.sign_out()
except Exception as x:
print(x)
client.sign_out()
exit(1)
def nebula2networkx(client: GraphClient, nebula_space: str,
graph: nx.MultiDiGraph, vertex_list, edge_types):
do_simple_execute(client, 'use ' + nebula_space)
yield_statement = ",".join(
[edge_type + "._dst" for edge_type in edge_types])
seen_vertex = set()
queue_vertex = []
all_edges = {}
for v in vertex_list:
queue_vertex.insert(0, v)
while len(queue_vertex):
vertex = queue_vertex.pop()
seen_vertex.add(vertex)
get_edge_go_statement = "GO FROM {} OVER {} YIELD ".format(
vertex, ','.join(edge_types)) + yield_statement
edges_resp = client.execute_query(get_edge_go_statement)
edges = [[] for _ in edge_types]
if edges_resp.rows is not None:
for row in edges_resp.rows:
for ids, col in enumerate(row.columns):
if (col.getType()
== ttypes.ColumnValue.ID) and col.get_id() != 0:
edges[ids].append(col.get_id())
if col.get_id() not in seen_vertex:
seen_vertex.add(col.get_id())
queue_vertex.insert(0, col.get_id())
all_edges[vertex] = edges
# build networkX graph Node
vertex_info_resp = fetch_info(client, "* " + str(vertex))
vertex_info = handle_fetch_resp(vertex_info_resp)
graph.add_node(vertex,
**vertex_info[0] if len(vertex_info) > 0 else {})
# build networkX graph Edge
for vertex_src, edges in all_edges.items():
for edge_type_ids, vertexs_dst in enumerate(edges):
if len(vertexs_dst) != 0:
edge_info_fetch_statement = edge_types[
edge_type_ids] + ' ' + ','.join([
str(vertex_src) + "->" + str(dst)
for dst in vertexs_dst
])
edges_info = handle_fetch_resp(
fetch_info(client, edge_info_fetch_statement))
graph.add_edges_from([(vertex_src, vertexs_dst[i],
edges_info[i])
for i in range(len(edges_info))])
def read_feats():
g_ip = '192.168.8.188'
g_port = 3699
connection_pool = ConnectionPool(g_ip, g_port)
client = GraphClient(connection_pool)
auth_resp = client.authenticate('root', '')
if auth_resp.error_code:
raise AuthException("Auth failed")
client.execute_query("use cora")
query_resp = client.execute_query("LOOKUP ON paper WHERE paper.num > 0")
num_nodes = len(query_resp.rows)
labels = np.empty((num_nodes, 1), dtype=np.int64)
label_map = {}
query_resp = client.execute_query("fetch prop ON paper 1")
feature_str = query_resp.rows[0].columns[2].get_str().decode().replace(
' ', '')
feat_data = np.zeros((num_nodes, len(feature_str)))
adj_lists = defaultdict(set)
for i in range(num_nodes):
query_resp = client.execute_query("fetch prop ON paper {}".format(i))
feature_str = query_resp.rows[0].columns[2].get_str().decode()
label = query_resp.rows[0].columns[3].get_str().decode()
feature = np.fromstring(str(feature_str), dtype=int, sep=' ')
feat_data[i] = feature
if not label in label_map:
label_map[label] = len(label_map)
labels[i] = label_map[label]
edge_query = client.execute_query("GO FROM {} OVER cite".format(i + 1))
if edge_query.rows is None:
continue
for row in edge_query.rows:
paperId = row.columns[0].get_id()
adj_lists[i].add(int(paperId))
adj_lists[int(paperId)].add(i)
return feat_data, labels, adj_lists
def create_client(arg: Arg) -> typing.Optional[GraphClient]:
connection_pool = ConnectionPool(arg.addr, arg.port)
client = GraphClient(connection_pool)
client.set_space('bgm')
try:
client.authenticate(arg.u, arg.p)
except AuthException as e:
print(e)
return None
return client
def test_response():
client = GraphClient(connection_pool)
auth_resp = client.authenticate('user', 'password')
if auth_resp.error_code:
raise AuthException("Auth failed")
query_resp = client.execute_query('SHOW SPACES')
do_simple_execute(client, 'use nba')
query_resp = client.execute_query(
'GO FROM 100 OVER follow,serve yield follow._dst,serve._dst')
query_resp = client.execute_query('fetch prop on * 100')
for row in query_resp.rows:
for ids, col in enumerate(row.columns):
print(col)
class Generator(object):
def __init__(self, ip, port):
connection_pool = ConnectionPool(ip, port, 1, 0)
self._client = GraphClient(connection_pool)
self._client.authenticate('root', 'nebula')
self._spaces = []
self._stmts = []
def start(self):
self.generate_space_statment()
self.generate_configs_statment()
def generate_space_statment(self):
resp = self.execute("SHOW SPACES;")
for row in resp.rows:
self._spaces.append(row.columns[0].get_str().decode('utf-8'))
if len(self._spaces) == 0:
return
for space in self._spaces:
self._stmts.append('\n')
self._stmts.append(
'# ============ Space[{}] ========='.format(space))
resp = self.execute("SHOW CREATE SPACE {};".format(space))
self._stmts.append(
resp.rows[0].columns[1].get_str().decode('utf-8').replace(
'\n', ''))
self.execute("USE {};".format(space))
self._stmts.append('USE {};'.format(space))
self.generate_tag_statment()
self.generate_edge_statment()
self.generate_tag_index_statment()
self.generate_edge_index_statment()
def generate_tag_statment(self):
resp = self.execute("SHOW TAGS;")
tags = []
for row in resp.rows:
tags.append(row.columns[1].get_str().decode('utf-8'))
if len(tags) == 0:
return
for tag in tags:
resp = self.execute("SHOW CREATE TAG {};".format(tag))
self._stmts.append(resp.rows[0].columns[1].get_str().decode(
'utf-8').replace('\n', '') + ';')
def generate_edge_statment(self):
resp = self.execute("SHOW EDGES;")
edges = []
for row in resp.rows:
edges.append(row.columns[1].get_str().decode('utf-8'))
if len(edges) == 0:
return
for edge in edges:
resp = self.execute("SHOW CREATE EDGE {};".format(edge))
self._stmts.append(resp.rows[0].columns[1].get_str().decode(
'utf-8').replace('\n', '') + ';')
def generate_tag_index_statment(self):
resp = self.execute("SHOW TAG INDEXES;")
tag_indexes = []
for row in resp.rows:
tag_indexes.append(row.columns[1].get_str().decode('utf-8'))
if len(tag_indexes) == 0:
return
for index in tag_indexes:
resp = self.execute("SHOW CREATE TAG INDEX {};".format(index))
self._stmts.append(resp.rows[0].columns[1].get_str().decode(
'utf-8').replace('\n', '') + ';')
def generate_edge_index_statment(self):
resp = self.execute("SHOW EDGE INDEXES;")
edge_indexes = []
for row in resp.rows:
edge_indexes.append(row.columns[1].get_str().decode('utf-8'))
if len(edge_indexes) == 0:
return
for index in edge_indexes:
resp = self.execute("SHOW CREATE EDGE INDEX {};".format(index))
self._stmts.append(resp.rows[0].columns[1].get_str().decode(
'utf-8').replace('\n', '') + ';')
def generate_configs_statment(self):
resp = self.execute("SHOW CONFIGS;")
# moduleName, configName, value
configs = []
for row in resp.rows:
module = row.columns[0].get_str().decode('utf-8')
config = row.columns[1].get_str().decode('utf-8')
col_val = row.columns[4]
if col_val.getType() == ttypes.ColumnValue.BOOL_VAL:
configs.append((module, config, col_val.get_bool_val()))
elif col_val.getType() == ttypes.ColumnValue.INTEGER:
configs.append((module, config, col_val.get_integer()))
elif col_val.getType() == ttypes.ColumnValue.STR:
configs.append(
(module, config, col_val.get_str().decode('utf-8').replace(
'\n', '').replace(':', '=').replace('"', '') + ';'))
elif col_val.getType() == ttypes.ColumnValue.DOUBLE_PRECISION:
configs.append(
(module, config, col_val.get_double_precision()))
else:
print("ERROR: Config {}:{} type `{}' unsupported".format(
module, config, col_val, col_val.getType()))
exit(1)
if len(configs) == 0:
return
self._stmts.append('\n')
self._stmts.append('# ============ Configs =========')
for config in configs:
self._stmts.append('UPDATE CONFIGS {}:{} = {}'.format(
config[0], config[1], config[2]))
def execute(self, stmt):
resp = self._client.execute_query(stmt)
if resp.error_code != ttypes.ErrorCode.SUCCEEDED:
print("Execute `SHOW SPACES' failed: ".format(resp.error_msg))
exit(1)
return resp
def print(self):
for stmt in self._stmts:
print(stmt)
def save_to_file(self):
f = open(statement_file, 'w')
for stmt in self._stmts:
f.write(stmt + '\n')
f.close()
print('The generated nGQLs file is ./{}'.format(statement_file))
def get_client(get_pool):
return GraphClient(get_pool)
def main_test():
client = None
global success_flag
try:
client = GraphClient(connection_pool)
if client.is_none():
print("ERROR: None client")
success_flag = False
return
space_name = 'space_' + threading.current_thread().getName()
resp = client.authenticate('user', 'password')
if resp.error_code != 0:
raise AuthException('Auth failed')
client.execute('DROP SPACE %s' % space_name)
resp = client.execute('CREATE SPACE %s' % space_name)
if resp.error_code != 0:
raise ExecutionException('CREATE SPACE failed')
resp = client.execute('USE %s' % space_name)
if resp.error_code != 0:
raise ExecutionException('USE SPACE failed')
client.sign_out()
except Exception as x:
print(x)
client.sign_out()
success_flag = False
return
def fetch_info(client: GraphClient, key: str):
return client.execute_query("FETCH PROP ON " + key)
def verify_nebula(client: GraphClient, datas: list, conf):
nebula_space = conf.get_string("nebula.space")
do_simple_execute(client, 'use ' + nebula_space)
batch_size = 50
full_conf = HandleConf(conf)
json_vertex_data = {}
json_edge_data = {}
for data in datas:
for tag_name, vertices in data["vertex"].items():
json_vertex_data[tag_name] = vertices["data"]
for edge_name, edge in data["edge"].items():
json_edge_data[edge_name] = edge["data"]
for tag_name, tag_conf in full_conf.tag_data.items():
logger.info("verify tag:" + tag_name)
data = json_vertex_data[tag_name]
query = 'fetch prop on {} '.format(tag_name)
ids = 0
while ids < len(data):
_ = 0
batch_ids = []
search_map = {}
while _ < batch_size and ids < len(data):
batch_ids.append(str(data[ids][tag_conf.id_field]))
search_map[str(data[ids][tag_conf.id_field])] = data[ids]
ids += 1
_ += 1
exec_query = query + ','.join(batch_ids)
logger.info("nebula exec: " + exec_query)
query_resp = client.execute_query(exec_query)
if query_resp.error_code != 0:
logger.error("resp msg:" + query_resp.error_msg)
else:
nebula_datas = handle_fetch_resp(query_resp)
if len(batch_ids) != len(nebula_datas):
logger.error("search length:{}, real length:{}".format(
len(batch_ids), len(nebula_datas)))
for nebula_data in nebula_datas:
json_data = search_map[str(nebula_data[str(
tag_conf.id_field)])]
for field in tag_conf.fields:
if nebula_data[field] != json_data[field]:
logger.error(
"json data:{} , nebula data:{} , field:{}, json value:{}, nebula value: {}",
json_data, nebula_data, field,
json_data[field], nebula_data[field])
def trim_edge_id(x):
return x if '.' not in x else x.split('.')[1]
for edge_name, edge_conf in full_conf.edge_data.items():
logger.info("verify edge:" + edge_name)
query = 'fetch prop on {} '.format(edge_name)
data = json_edge_data[edge_name]
for json_data in data:
edge_id = edge_conf.get_id(
json_data['from']['match'][trim_edge_id(edge_conf.from_field)],
json_data['to']['match'][trim_edge_id(edge_conf.to_field)],
0 if edge_conf.ranking == '' else
json_data['data'][edge_conf.ranking])
exec_query = query + edge_id
logger.info("nebula exec: " + exec_query)
query_resp = client.execute_query(exec_query)
if query_resp.error_code != 0:
logger.error("resp msg: " + query_resp.error_msg)
else:
nebula_datas = handle_fetch_resp(query_resp)
if len(nebula_datas) != 1:
logger.error("nebula data:{}, json_data:{}".format(
nebula_datas, json_data))
else:
nebula_data = nebula_datas[0]
for field in edge_conf.fields:
if field not in nebula_data:
logger.error("field:{} not in nebula data{}",
field, nebula_data)
if nebula_data[field] != json_data['data'][field]:
logger.error(
"json data:{} , nebula data:{} , field:{}, json value:{}, nebula value: {}",
json_data, nebula_data, field,
json_data[field], nebula_data[field])
if edge_conf.ranking != '':
if nebula_data['_rank'] != json_data['data'][
edge_conf.ranking]:
logger.error(
"rank field {}, nebula rank {}, json rank {}".
format(edge_conf.ranking, nebula_data['_rank'],
json_data['data'][edge_conf.ranking]))
def test_prepare():
try:
client = GraphClient(ConnectionPool(host, graph_port))
if client is None:
print('Error: None GraphClient')
assert False
return
resp = client.authenticate('user', 'password')
assert resp.error_code == 0, resp.error_msg
resp = client.execute('DROP SPACE IF EXISTS %s' % space_name)
assert resp.error_code == 0, resp.error_msg
resp = client.execute('CREATE SPACE %s(partition_num=1)' % space_name)
assert resp.error_code == 0, resp.error_msg
time.sleep(5)
resp = client.execute('USE %s' % space_name)
assert resp.error_code == 0, resp.error_msg
time.sleep(5)
resp = client.execute(
'CREATE TAG player(name string, age int, married bool)')
assert resp.error_code == 0, resp.error_msg
resp = client.execute('CREATE TAG team(name string, money double)')
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'CREATE EDGE follow(degree double, likeness int)')
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'CREATE EDGE serve(start timestamp, end timestamp)')
assert resp.error_code == 0, resp.error_msg
time.sleep(12)
resp = client.execute(
'INSERT VERTEX player(name, age, married) VALUES 101:(\'Bob\', 18, FALSE)'
)
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT VERTEX player(name, age, married) VALUES 102:(\'Tom\', 22, TRUE)'
)
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT VERTEX player(name, age, married) VALUES 103:(\'Jerry\', 19, FALSE)'
)
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT VERTEX team(name, money) VALUES 201:(\'Red Bull\', 185.85)'
)
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT VERTEX team(name, money) VALUES 202:(\'River\', 567.93)')
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT EDGE follow(degree, likeness) VALUES 101->102:(94.7, 45)')
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT EDGE follow(degree, likeness) VALUES 102->103:(86.3, 79)')
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT EDGE serve(start, end) VALUES 101->201:(\'2001-09-01 08:00:00\', \'2010-09-01 08:00:00\')'
)
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT EDGE serve(start, end) VALUES 102->202:(\'1998-08-22 06:45:54\', \'2020-01-23 17:23:35\')'
)
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT EDGE serve(start, end) VALUES 103->201:(\'2006-11-18 13:28:29\', \'2009-12-12 12:21:46\')'
)
assert resp.error_code == 0, resp.error_msg
except Exception as ex:
print(ex)
client.sign_out()
assert False
from graph import ttypes
from nebula.ConnectionPool import ConnectionPool
from nebula.Client import GraphClient
from nebula.Common import *
g_ip = '192.168.8.188'
g_port = 3699
connection_pool = ConnectionPool(g_ip, g_port)
client = GraphClient(connection_pool)
auth_resp = client.authenticate('root', '')
if auth_resp.error_code:
raise AuthException("Auth failed")
import numpy as np
from collections import defaultdict
def read_feats(client):
client.execute_query("use cora")
query_resp = client.execute_query("LOOKUP ON paper WHERE paper.num > 0")
num_nodes=len(query_resp.rows)
labels = np.empty((num_nodes,1), dtype=np.int64)
label_map = {}
query_resp = client.execute_query("fetch prop ON paper 1")
feature_str = query_resp.rows[0].columns[2].get_str().decode().replace(' ', '')
feat_data = np.zeros((num_nodes, len(feature_str)))
adj_lists = defaultdict(set)
for i in range(num_nodes):
query_resp = client.execute_query("fetch prop ON paper {}".format(i))
def create_schema(client: GraphClient):
resp = client.execute(SCHEMA)
if resp.error_code != 0:
print(f"schema create failure {resp.error_msg}")
def __init__(self, ip, port):
connection_pool = ConnectionPool(ip, port, 1, 0)
self._client = GraphClient(connection_pool)
self._client.authenticate('root', 'nebula')
self._spaces = []
self._stmts = []
'--config',
help='spark importer config path',
type=str,
default="application.conf")
parser.add_argument(
'-l',
'--log_level',
help='log level=> 10:debug, 20:info, 30:warn, 40:error',
type=int,
default=30)
args = parser.parse_args()
connection_pool = ConnectionPool(
args.address.split(":")[0],
args.address.split(":")[1])
client = GraphClient(connection_pool)
auth_resp = client.authenticate(args.user, args.password)
if auth_resp.error_code:
raise AuthException("Auth failed")
conf = ConfigFactory.parse_file(args.config)
data = []
for j in args.jsons:
with open(j) as f:
data.append(json.load(f))
logger.setLevel(args.log_level)
verify_nebula(client, data, conf)
if len(sys.argv) == 3:
print('input argv num is 3')
g_ip = sys.argv[1]
print('ip: %s' % g_ip)
g_port = sys.argv[2]
print('port: %s' % g_port)
# init connection pool
# if your tags or edges' num is more than 10,
# please make sure conn_num >= max(tags_num, edges_num) * 2
conn_num = 20
connection_pool = ConnectionPool(g_ip, g_port, conn_num)
clients = []
for i in range(0, conn_num):
client = GraphClient(connection_pool)
client.authenticate('user', 'password')
client.execute('USE test')
clients.append(client)
# Get client
resp = clients[0].execute_query('SHOW TAGS')
if resp.error_code:
raise ExecutionException('SHOW TAGS failed')
tag_list = []
for row in resp.rows:
tag_list.append(row.columns[1].get_str().decode('utf-8'))
resp = clients[0].execute_query('SHOW EDGES')
if resp.error_code:
raise ExecutionException('SHOW EDGES failed')
str(vertex_src) + "->" + str(dst)
for dst in vertexs_dst
])
edges_info = handle_fetch_resp(
fetch_info(client, edge_info_fetch_statement))
graph.add_edges_from([(vertex_src, vertexs_dst[i],
edges_info[i])
for i in range(len(edges_info))])
def draw_networkx_graph(graph):
import matplotlib.pyplot as plt
nx.draw(graph, with_labels=True, font_weight='bold')
plt.show()
if __name__ == '__main__':
g_ip = '127.0.0.1'
g_port = 3699
# init connection pool
connection_pool = ConnectionPool(g_ip, g_port)
G = nx.MultiDiGraph()
client = GraphClient(connection_pool)
auth_resp = client.authenticate('user', 'password')
if auth_resp.error_code:
raise AuthException("Auth failed")
nebula2networkx(client, 'nba', G, [100], ['follow', 'serve'])
draw_networkx_graph(G)
