def create_client(arg: Arg) -> typing.Optional[GraphClient]:
connection_pool = ConnectionPool(arg.addr, arg.port)
client = GraphClient(connection_pool)
client.set_space('bgm')
try:
client.authenticate(arg.u, arg.p)
except AuthException as e:
print(e)
return None
return client
def main_test():
client = None
global success_flag
try:
client = GraphClient(connection_pool)
if client.is_none():
print("ERROR: None client")
success_flag = False
return
space_name = 'space_' + threading.current_thread().getName()
resp = client.authenticate('user', 'password')
if resp.error_code != 0:
raise AuthException('Auth failed')
client.execute('DROP SPACE %s' % space_name)
resp = client.execute('CREATE SPACE %s' % space_name)
if resp.error_code != 0:
raise ExecutionException('CREATE SPACE failed')
resp = client.execute('USE %s' % space_name)
if resp.error_code != 0:
raise ExecutionException('USE SPACE failed')
client.sign_out()
except Exception as x:
print(x)
client.sign_out()
success_flag = False
return
def main_test():
client = None
try:
space_name = 'space_' + threading.current_thread().getName()
print('thread name: %s, space_name : %s' %
(threading.current_thread().getName(), space_name))
# Get one client
client = GraphClient(connection_pool)
auth_resp = client.authenticate('user', 'password')
if auth_resp.error_code:
raise AuthException("Auth failed")
query_resp = client.execute_query('SHOW SPACES')
if has_space(query_resp.rows, space_name):
print('has %s, drop it' % space_name)
do_simple_execute(client, 'DROP SPACE %s' % space_name)
# Create space mySpace
do_simple_execute(client, 'CREATE SPACE %s'
% space_name)
do_simple_execute(client, 'USE %s' % space_name)
time.sleep(1)
# Create tag and edge
do_simple_execute(client, 'CREATE TAG person(name string, age int); '
'CREATE EDGE like(likeness double)')
# It should large than the cycle of loading the schema
time.sleep(6)
# Insert vertex and edge
do_simple_execute(client, 'INSERT VERTEX person(name, age) VALUES 1:(\'Bob\', 10)')
do_simple_execute(client, 'INSERT VERTEX person(name, age) VALUES 2:(\'Lily\', 9)')
do_simple_execute(client, 'INSERT VERTEX person(name, age) VALUES 3:(\'Tom\', 10)')
do_simple_execute(client, 'INSERT VERTEX person(name, age) VALUES 4:(\'Jerry\', 13);INSERT VERTEX person(name, age) VALUES 5:(\'John\', 11)')
do_simple_execute(client, 'INSERT EDGE like(likeness) VALUES 1->2:(80.0)')
do_simple_execute(client, 'INSERT EDGE like(likeness) VALUES 1->3:(70.0)')
do_simple_execute(client, 'INSERT EDGE like(likeness) VALUES 2->4:(84.0), 3->5:(68.3), 1->5:(97.2)')
# Query data
query_resp = client.execute_query('GO FROM 1 OVER like YIELD $$.person.name, '
'$$.person.age, like.likeness')
if query_resp.error_code:
print('Execute failed: %s' % query_resp.error_msg)
exit(1)
# Print the result of query
print(' \n====== The query result of thread[%s]======\n '
% threading.current_thread().getName())
print_value(query_resp.column_names, query_resp.rows)
client.sign_out()
except Exception as x:
print(x)
client.sign_out()
exit(1)
def test_response():
client = GraphClient(connection_pool)
auth_resp = client.authenticate('user', 'password')
if auth_resp.error_code:
raise AuthException("Auth failed")
query_resp = client.execute_query('SHOW SPACES')
do_simple_execute(client, 'use nba')
query_resp = client.execute_query(
'GO FROM 100 OVER follow,serve yield follow._dst,serve._dst')
query_resp = client.execute_query('fetch prop on * 100')
for row in query_resp.rows:
for ids, col in enumerate(row.columns):
print(col)
def read_feats():
g_ip = '192.168.8.188'
g_port = 3699
connection_pool = ConnectionPool(g_ip, g_port)
client = GraphClient(connection_pool)
auth_resp = client.authenticate('root', '')
if auth_resp.error_code:
raise AuthException("Auth failed")
client.execute_query("use cora")
query_resp = client.execute_query("LOOKUP ON paper WHERE paper.num > 0")
num_nodes = len(query_resp.rows)
labels = np.empty((num_nodes, 1), dtype=np.int64)
label_map = {}
query_resp = client.execute_query("fetch prop ON paper 1")
feature_str = query_resp.rows[0].columns[2].get_str().decode().replace(
' ', '')
feat_data = np.zeros((num_nodes, len(feature_str)))
adj_lists = defaultdict(set)
for i in range(num_nodes):
query_resp = client.execute_query("fetch prop ON paper {}".format(i))
feature_str = query_resp.rows[0].columns[2].get_str().decode()
label = query_resp.rows[0].columns[3].get_str().decode()
feature = np.fromstring(str(feature_str), dtype=int, sep=' ')
feat_data[i] = feature
if not label in label_map:
label_map[label] = len(label_map)
labels[i] = label_map[label]
edge_query = client.execute_query("GO FROM {} OVER cite".format(i + 1))
if edge_query.rows is None:
continue
for row in edge_query.rows:
paperId = row.columns[0].get_id()
adj_lists[i].add(int(paperId))
adj_lists[int(paperId)].add(i)
return feat_data, labels, adj_lists
print('input argv num is 3')
g_ip = sys.argv[1]
print('ip: %s' % g_ip)
g_port = sys.argv[2]
print('port: %s' % g_port)
# init connection pool
# if your tags or edges' num is more than 10,
# please make sure conn_num >= max(tags_num, edges_num) * 2
conn_num = 20
connection_pool = ConnectionPool(g_ip, g_port, conn_num)
clients = []
for i in range(0, conn_num):
client = GraphClient(connection_pool)
client.authenticate('user', 'password')
client.execute('USE test')
clients.append(client)
# Get client
resp = clients[0].execute_query('SHOW TAGS')
if resp.error_code:
raise ExecutionException('SHOW TAGS failed')
tag_list = []
for row in resp.rows:
tag_list.append(row.columns[1].get_str().decode('utf-8'))
resp = clients[0].execute_query('SHOW EDGES')
if resp.error_code:
raise ExecutionException('SHOW EDGES failed')
class Generator(object):
def __init__(self, ip, port):
connection_pool = ConnectionPool(ip, port, 1, 0)
self._client = GraphClient(connection_pool)
self._client.authenticate('root', 'nebula')
self._spaces = []
self._stmts = []
def start(self):
self.generate_space_statment()
self.generate_configs_statment()
def generate_space_statment(self):
resp = self.execute("SHOW SPACES;")
for row in resp.rows:
self._spaces.append(row.columns[0].get_str().decode('utf-8'))
if len(self._spaces) == 0:
return
for space in self._spaces:
self._stmts.append('\n')
self._stmts.append(
'# ============ Space[{}] ========='.format(space))
resp = self.execute("SHOW CREATE SPACE {};".format(space))
self._stmts.append(
resp.rows[0].columns[1].get_str().decode('utf-8').replace(
'\n', ''))
self.execute("USE {};".format(space))
self._stmts.append('USE {};'.format(space))
self.generate_tag_statment()
self.generate_edge_statment()
self.generate_tag_index_statment()
self.generate_edge_index_statment()
def generate_tag_statment(self):
resp = self.execute("SHOW TAGS;")
tags = []
for row in resp.rows:
tags.append(row.columns[1].get_str().decode('utf-8'))
if len(tags) == 0:
return
for tag in tags:
resp = self.execute("SHOW CREATE TAG {};".format(tag))
self._stmts.append(resp.rows[0].columns[1].get_str().decode(
'utf-8').replace('\n', '') + ';')
def generate_edge_statment(self):
resp = self.execute("SHOW EDGES;")
edges = []
for row in resp.rows:
edges.append(row.columns[1].get_str().decode('utf-8'))
if len(edges) == 0:
return
for edge in edges:
resp = self.execute("SHOW CREATE EDGE {};".format(edge))
self._stmts.append(resp.rows[0].columns[1].get_str().decode(
'utf-8').replace('\n', '') + ';')
def generate_tag_index_statment(self):
resp = self.execute("SHOW TAG INDEXES;")
tag_indexes = []
for row in resp.rows:
tag_indexes.append(row.columns[1].get_str().decode('utf-8'))
if len(tag_indexes) == 0:
return
for index in tag_indexes:
resp = self.execute("SHOW CREATE TAG INDEX {};".format(index))
self._stmts.append(resp.rows[0].columns[1].get_str().decode(
'utf-8').replace('\n', '') + ';')
def generate_edge_index_statment(self):
resp = self.execute("SHOW EDGE INDEXES;")
edge_indexes = []
for row in resp.rows:
edge_indexes.append(row.columns[1].get_str().decode('utf-8'))
if len(edge_indexes) == 0:
return
for index in edge_indexes:
resp = self.execute("SHOW CREATE EDGE INDEX {};".format(index))
self._stmts.append(resp.rows[0].columns[1].get_str().decode(
'utf-8').replace('\n', '') + ';')
def generate_configs_statment(self):
resp = self.execute("SHOW CONFIGS;")
# moduleName, configName, value
configs = []
for row in resp.rows:
module = row.columns[0].get_str().decode('utf-8')
config = row.columns[1].get_str().decode('utf-8')
col_val = row.columns[4]
if col_val.getType() == ttypes.ColumnValue.BOOL_VAL:
configs.append((module, config, col_val.get_bool_val()))
elif col_val.getType() == ttypes.ColumnValue.INTEGER:
configs.append((module, config, col_val.get_integer()))
elif col_val.getType() == ttypes.ColumnValue.STR:
configs.append(
(module, config, col_val.get_str().decode('utf-8').replace(
'\n', '').replace(':', '=').replace('"', '') + ';'))
elif col_val.getType() == ttypes.ColumnValue.DOUBLE_PRECISION:
configs.append(
(module, config, col_val.get_double_precision()))
else:
print("ERROR: Config {}:{} type `{}' unsupported".format(
module, config, col_val, col_val.getType()))
exit(1)
if len(configs) == 0:
return
self._stmts.append('\n')
self._stmts.append('# ============ Configs =========')
for config in configs:
self._stmts.append('UPDATE CONFIGS {}:{} = {}'.format(
config[0], config[1], config[2]))
def execute(self, stmt):
resp = self._client.execute_query(stmt)
if resp.error_code != ttypes.ErrorCode.SUCCEEDED:
print("Execute `SHOW SPACES' failed: ".format(resp.error_msg))
exit(1)
return resp
def print(self):
for stmt in self._stmts:
print(stmt)
def save_to_file(self):
f = open(statement_file, 'w')
for stmt in self._stmts:
f.write(stmt + '\n')
f.close()
print('The generated nGQLs file is ./{}'.format(statement_file))
'--config',
help='spark importer config path',
type=str,
default="application.conf")
parser.add_argument(
'-l',
'--log_level',
help='log level=> 10:debug, 20:info, 30:warn, 40:error',
type=int,
default=30)
args = parser.parse_args()
connection_pool = ConnectionPool(
args.address.split(":")[0],
args.address.split(":")[1])
client = GraphClient(connection_pool)
auth_resp = client.authenticate(args.user, args.password)
if auth_resp.error_code:
raise AuthException("Auth failed")
conf = ConfigFactory.parse_file(args.config)
data = []
for j in args.jsons:
with open(j) as f:
data.append(json.load(f))
logger.setLevel(args.log_level)
verify_nebula(client, data, conf)
def test_prepare():
try:
client = GraphClient(ConnectionPool(host, graph_port))
if client is None:
print('Error: None GraphClient')
assert False
return
resp = client.authenticate('user', 'password')
assert resp.error_code == 0, resp.error_msg
resp = client.execute('DROP SPACE IF EXISTS %s' % space_name)
assert resp.error_code == 0, resp.error_msg
resp = client.execute('CREATE SPACE %s(partition_num=1)' % space_name)
assert resp.error_code == 0, resp.error_msg
time.sleep(5)
resp = client.execute('USE %s' % space_name)
assert resp.error_code == 0, resp.error_msg
time.sleep(5)
resp = client.execute(
'CREATE TAG player(name string, age int, married bool)')
assert resp.error_code == 0, resp.error_msg
resp = client.execute('CREATE TAG team(name string, money double)')
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'CREATE EDGE follow(degree double, likeness int)')
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'CREATE EDGE serve(start timestamp, end timestamp)')
assert resp.error_code == 0, resp.error_msg
time.sleep(12)
resp = client.execute(
'INSERT VERTEX player(name, age, married) VALUES 101:(\'Bob\', 18, FALSE)'
)
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT VERTEX player(name, age, married) VALUES 102:(\'Tom\', 22, TRUE)'
)
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT VERTEX player(name, age, married) VALUES 103:(\'Jerry\', 19, FALSE)'
)
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT VERTEX team(name, money) VALUES 201:(\'Red Bull\', 185.85)'
)
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT VERTEX team(name, money) VALUES 202:(\'River\', 567.93)')
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT EDGE follow(degree, likeness) VALUES 101->102:(94.7, 45)')
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT EDGE follow(degree, likeness) VALUES 102->103:(86.3, 79)')
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT EDGE serve(start, end) VALUES 101->201:(\'2001-09-01 08:00:00\', \'2010-09-01 08:00:00\')'
)
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT EDGE serve(start, end) VALUES 102->202:(\'1998-08-22 06:45:54\', \'2020-01-23 17:23:35\')'
)
assert resp.error_code == 0, resp.error_msg
resp = client.execute(
'INSERT EDGE serve(start, end) VALUES 103->201:(\'2006-11-18 13:28:29\', \'2009-12-12 12:21:46\')'
)
assert resp.error_code == 0, resp.error_msg
except Exception as ex:
print(ex)
client.sign_out()
assert False
from graph import ttypes
from nebula.ConnectionPool import ConnectionPool
from nebula.Client import GraphClient
from nebula.Common import *
g_ip = '192.168.8.188'
g_port = 3699
connection_pool = ConnectionPool(g_ip, g_port)
client = GraphClient(connection_pool)
auth_resp = client.authenticate('root', '')
if auth_resp.error_code:
raise AuthException("Auth failed")
import numpy as np
from collections import defaultdict
def read_feats(client):
client.execute_query("use cora")
query_resp = client.execute_query("LOOKUP ON paper WHERE paper.num > 0")
num_nodes=len(query_resp.rows)
labels = np.empty((num_nodes,1), dtype=np.int64)
label_map = {}
query_resp = client.execute_query("fetch prop ON paper 1")
feature_str = query_resp.rows[0].columns[2].get_str().decode().replace(' ', '')
feat_data = np.zeros((num_nodes, len(feature_str)))
adj_lists = defaultdict(set)
for i in range(num_nodes):
query_resp = client.execute_query("fetch prop ON paper {}".format(i))
str(vertex_src) + "->" + str(dst)
for dst in vertexs_dst
])
edges_info = handle_fetch_resp(
fetch_info(client, edge_info_fetch_statement))
graph.add_edges_from([(vertex_src, vertexs_dst[i],
edges_info[i])
for i in range(len(edges_info))])
def draw_networkx_graph(graph):
import matplotlib.pyplot as plt
nx.draw(graph, with_labels=True, font_weight='bold')
plt.show()
if __name__ == '__main__':
g_ip = '127.0.0.1'
g_port = 3699
# init connection pool
connection_pool = ConnectionPool(g_ip, g_port)
G = nx.MultiDiGraph()
client = GraphClient(connection_pool)
auth_resp = client.authenticate('user', 'password')
if auth_resp.error_code:
raise AuthException("Auth failed")
nebula2networkx(client, 'nba', G, [100], ['follow', 'serve'])
draw_networkx_graph(G)