def test_init_failed(self):
# init succeeded
pool1 = ConnectionPool()
addresses = list()
addresses.append(('127.0.0.1', 9669))
addresses.append(('127.0.0.1', 9670))
assert pool1.init(addresses, Config())
# init failed, connected failed
pool2 = ConnectionPool()
addresses = list()
addresses.append(('127.0.0.1', 3800))
try:
pool2.init(addresses, Config())
assert False
except Exception:
assert True
# init failed, hostname not existed
try:
pool3 = ConnectionPool()
addresses = list()
addresses.append(('not_exist_hostname', 3800))
assert not pool3.init(addresses, Config())
except InValidHostname:
assert True, "We expected get the exception"
def test_multi_thread():
# Test multi thread
addresses = [('127.0.0.1', 9669), ('127.0.0.1', 9670)]
configs = Config()
configs.max_connection_pool_size = 4
pool = ConnectionPool()
assert pool.init(addresses, configs)
global success_flag
success_flag = True
def main_test():
session = None
global success_flag
try:
session = pool.get_session('root', 'nebula')
if session is None:
success_flag = False
return
space_name = 'space_' + threading.current_thread().getName()
session.execute('DROP SPACE %s' % space_name)
resp = session.execute(
'CREATE SPACE IF NOT EXISTS %s(vid_type=FIXED_STRING(8))' %
space_name)
if not resp.is_succeeded():
raise RuntimeError('CREATE SPACE failed: {}'.format(
resp.error_msg()))
time.sleep(3)
resp = session.execute('USE %s' % space_name)
if not resp.is_succeeded():
raise RuntimeError('USE SPACE failed:{}'.format(
resp.error_msg()))
except Exception as x:
print(x)
success_flag = False
return
finally:
if session is not None:
session.release()
thread1 = threading.Thread(target=main_test, name='thread1')
thread2 = threading.Thread(target=main_test, name='thread2')
thread3 = threading.Thread(target=main_test, name='thread3')
thread4 = threading.Thread(target=main_test, name='thread4')
thread1.start()
thread2.start()
thread3.start()
thread4.start()
thread1.join()
thread2.join()
thread3.join()
thread4.join()
pool.close()
assert success_flag
def start(self):
os.chdir(self.work_dir)
start_time = time.time()
for p in self.all_processes:
p.start()
time.sleep(3)
config = Config()
config.max_connection_pool_size = 20
config.timeout = 60000
# init connection pool
client_pool = ConnectionPool()
# assert client_pool.init([("127.0.0.1", int(self.graphd_port))], config)
assert client_pool.init([("127.0.0.1", self.graphd_processes[0].tcp_port)], config)
cmd = "ADD HOSTS 127.0.0.1:" + str(self.storaged_processes[0].tcp_port) + " INTO NEW ZONE \"default_zone\""
print(cmd)
# get session from the pool
client = client_pool.get_session('root', 'nebula')
resp = client.execute(cmd)
client.release()
# wait nebula start
server_ports = [p.tcp_port for p in self.all_processes]
if not self._check_servers_status(server_ports):
self._collect_pids()
self.kill_all(signal.SIGKILL)
elapse = time.time() - start_time
raise Exception(f'nebula servers not ready in {elapse}s')
self._collect_pids()
return [p.tcp_port for p in self.graphd_processes]
class TestSession(TestCase):
@classmethod
def setup_class(self):
self.addresses = list()
self.addresses.append(('127.0.0.1', 3699))
self.addresses.append(('127.0.0.1', 3700))
self.user_name = 'root'
self.password = '******'
self.configs = Config()
self.configs.min_connection_pool_size = 2
self.configs.max_connection_pool_size = 4
self.pool = ConnectionPool()
self.pool._check_delay = 2
assert self.pool.init(self.addresses, self.configs)
assert self.pool.connnects() == 2
def test_reconnect(self):
try:
import time
session = self.pool.get_session('root', 'nebula')
for i in range(0, 30):
session.execute('SHOW SPACES')
time.sleep(2)
new_session = self.pool.get_session('root', 'nebula')
new_session.execute('SHOW SPACES')
except Exception:
assert False
def test_wrong_hostname(self):
pool = ConnectionPool()
try:
pool.init([('wrong_host', 9669)], Config())
assert False
except InValidHostname:
assert True
def get_conn_pool(host: str, port: int):
config = Config()
config.max_connection_pool_size = 20
config.timeout = 180000
# init connection pool
pool = ConnectionPool()
if not pool.init([(host, port)], config):
raise Exception("Fail to init connection pool.")
return pool
def setup_class(self):
self.user_name = 'root'
self.password = '******'
self.configs = Config()
self.configs.max_connection_pool_size = 6
self.pool = ConnectionPool()
assert self.pool.init([('127.0.0.1', 9669), ('127.0.0.1', 9670),
('127.0.0.1', 9671)], self.configs)
assert self.pool.connnects() == 0
assert self.pool.in_used_connects() == 0
def setup_class(self):
self.addresses = list()
self.addresses.append(('127.0.0.1', 3699))
self.addresses.append(('127.0.0.1', 3700))
self.configs = Config()
self.configs.min_connection_pool_size = 2
self.configs.max_connection_pool_size = 4
self.pool = ConnectionPool()
assert self.pool.init(self.addresses, self.configs)
assert self.pool.connnects() == 2
def create_nebula_clients(self):
config = Config()
config.max_connection_pool_size = 20
config.timeout = 60000
# init connection pool
self.client_pool = ConnectionPool()
assert self.client_pool.init([(self.host, self.port)], config)
# get session from the pool
self.client = self.client_pool.get_session(self.user, self.password)
def start(self):
os.chdir(self.work_dir)
start_time = time.time()
for p in self.all_processes:
p.start()
config = Config()
config.max_connection_pool_size = 20
config.timeout = 60000
# init connection pool
client_pool = ConnectionPool()
# assert client_pool.init([("127.0.0.1", int(self.graphd_port))], config)
ssl_config = get_ssl_config(self.is_graph_ssl, self.ca_signed)
print("begin to add hosts")
ok = False
# wait graph is ready, and then add hosts
for _ in range(20):
try:
ok = client_pool.init(
[("127.0.0.1", self.graphd_processes[0].tcp_port)],
config,
ssl_config,
)
if ok:
break
except:
pass
time.sleep(1)
assert ok, "graph is not ready"
# get session from the pool
client = client_pool.get_session('root', 'nebula')
hosts = ",".join([
"127.0.0.1:{}".format(str(storaged.tcp_port))
for storaged in self.storaged_processes
])
cmd = "ADD HOSTS {} INTO NEW ZONE \"default_zone\"".format(hosts)
print("add hosts cmd is {}".format(cmd))
resp = client.execute(cmd)
assert resp.is_succeeded(), resp.error_msg()
client.release()
# wait nebula start
server_ports = [p.tcp_port for p in self.all_processes]
if not self._check_servers_status(server_ports):
self._collect_pids()
self.kill_all(signal.SIGKILL)
elapse = time.time() - start_time
raise Exception(f'nebula servers not ready in {elapse}s')
self._collect_pids()
return [p.tcp_port for p in self.graphd_processes]
def __enter__(self):
config = Config()
config.max_connection_pool_size = 2
config.timeout = 0
# init connection pool
self.client_pool = ConnectionPool()
assert self.client_pool.init([(self.host, self.port)], config)
# get session from the pool
self.client = self.client_pool.get_session(self.user, self.password)
return self
def setup_class(self):
self.addresses = list()
self.addresses.append(('127.0.0.1', 9669))
self.addresses.append(('127.0.0.1', 9670))
self.configs = Config()
self.configs.min_connection_pool_size = 2
self.configs.max_connection_pool_size = 4
self.configs.idle_time = 2000
self.configs.interval_check = 2
self.pool = ConnectionPool()
assert self.pool.init(self.addresses, self.configs)
assert self.pool.connnects() == 2
def setup_class(self):
self.addresses = list()
self.addresses.append(('127.0.0.1', 3699))
self.addresses.append(('127.0.0.1', 3700))
self.user_name = 'root'
self.password = '******'
self.configs = Config()
self.configs.min_connection_pool_size = 2
self.configs.max_connection_pool_size = 4
self.pool = ConnectionPool()
self.pool._check_delay = 2
assert self.pool.init(self.addresses, self.configs)
assert self.pool.connnects() == 2
class TestSession(TestCase):
@classmethod
def setup_class(self):
self.user_name = 'root'
self.password = '******'
self.configs = Config()
self.configs.max_connection_pool_size = 6
self.pool = ConnectionPool()
assert self.pool.init([('127.0.0.1', 9669), ('127.0.0.1', 9670),
('127.0.0.1', 9671)], self.configs)
assert self.pool.connnects() == 0
assert self.pool.in_used_connects() == 0
def test_1_release_by_del(self):
def get_local_session(pool):
session = pool.get_session('root', 'nebula')
assert pool.in_used_connects() == 1
get_local_session(self.pool)
assert self.pool.in_used_connects() == 0
def test_2_reconnect(self):
try:
session = self.pool.get_session('root', 'nebula')
session.execute(
'CREATE SPACE IF NOT EXISTS test_session(vid_type=FIXED_STRING(8)); USE test_session;'
)
for i in range(0, 5):
if i == 3:
os.system('docker stop nebula-docker-compose_graphd0_1')
os.system('docker stop nebula-docker-compose_graphd1_1')
time.sleep(3)
resp = session.execute('SHOW TAGS')
assert resp.is_succeeded()
assert resp.space_name() == 'test_session'
time.sleep(2)
session.release()
new_session = self.pool.get_session('root', 'nebula')
new_session.execute('SHOW SPACES')
except Exception as e:
assert False, e
finally:
os.system('docker start nebula-docker-compose_graphd0_1')
os.system('docker start nebula-docker-compose_graphd1_1')
time.sleep(5)
def test_3_session_context(self):
in_used_connects = self.pool.in_used_connects()
with self.pool.session_context('root', 'nebula') as session:
assert self.pool.in_used_connects() == in_used_connects + 1
assert self.pool.in_used_connects() == in_used_connects
def __init__(self):
# define a config
config = Config()
config.max_connection_pool_size = 10
# init connection pool
connection_pool = ConnectionPool()
# if the given servers are ok, return true, else return false
ok = connection_pool.init([('10.141.186.105', 3699)], config)
# option 1 control the connection release yourself
# get session from the pool
self.session = connection_pool.get_session('root', 'nebula')
# select space
self.session.execute('USE adeci_test')
def test_4_timeout(self):
try:
configs = Config()
configs.timeout = 100
configs.max_connection_pool_size = 1
pool = ConnectionPool()
assert pool.init([('127.0.0.1', 9669)], configs)
session = pool.get_session(self.user_name, self.password)
ngql = ''
for n in range(0, 500):
ngql = ngql + 'show hosts;'
session.execute(ngql)
assert False, 'expect to get exception'
except Exception as ex:
assert str(ex).find('timed out') > 0
assert True, ex
def load_csv_data_once(
tmp_path_factory,
pytestconfig,
worker_id,
conn_pool: ConnectionPool,
space: str,
):
root_tmp_dir = tmp_path_factory.getbasetemp().parent
fn = root_tmp_dir / f"csv-data-{space}"
is_file = True
with FileLock(str(fn) + ".lock"):
if not fn.is_file():
data_dir = os.path.join(CURR_PATH, "data", space)
user = pytestconfig.getoption("user")
password = pytestconfig.getoption("password")
sess = conn_pool.get_session(user, password)
space_desc = load_csv_data(pytestconfig, sess, data_dir)
sess.release()
fn.write_text(json.dumps(space_desc.__dict__))
is_file = False
else:
space_desc = SpaceDesc.from_json(json.loads(fn.read_text()))
if is_file:
logging.info(f"session-{worker_id} need not to load {space} csv data")
yield space_desc
else:
logging.info(f"session-{worker_id} load {space} csv data")
yield space_desc
os.remove(str(fn))
def test_timeout(self):
config = Config()
config.timeout = 1000
config.max_connection_pool_size = 1
pool = ConnectionPool()
assert pool.init([('127.0.0.1', 9669)], config)
session = pool.get_session('root', 'nebula')
try:
resp = session.execute(
'USE nba;GO 1000 STEPS FROM \"Tim Duncan\" OVER like')
assert False
except IOErrorException as e:
assert True
assert str(e).find("Read timed out")
session.release()
try:
session = pool.get_session('root', 'nebula')
except IOErrorException as e:
assert False
def prepare_data():
config = Config()
config.max_connection_pool_size = 1
# init connection pool
connection_pool = ConnectionPool()
# the graphd server's address
assert connection_pool.init([('172.28.3.1', 3699)], config)
client = connection_pool.get_session('root', 'nebula')
client.execute('CREATE SPACE IF NOT EXISTS ScanSpace('
'PARTITION_NUM=10,'
'vid_type=FIXED_STRING(20));'
'USE ScanSpace;'
'CREATE TAG IF NOT EXISTS person(name string, age int);'
'CREATE EDGE IF NOT EXISTS friend(start int, end int);')
time.sleep(5)
for id in range(20):
vid = 'person' + str(id)
cmd = 'INSERT VERTEX person(name, age) ' \
'VALUES \"{}\":(\"{}\", {})'.format(vid, vid, id)
client.execute(cmd)
for id in range(20):
src_id = 'person' + str(id)
dst_id = 'person' + str(20 - id)
start = random.randint(2000, 2010)
end = random.randint(2010, 2020)
cmd = 'INSERT EDGE friend(start, end) ' \
'VALUES \"{}\"->\"{}\":({}, {})'.format(src_id, dst_id, start, end)
client.execute(cmd)
client.release()
connection_pool.close()
class TestSession(TestCase):
@classmethod
def setup_class(self):
self.user_name = 'root'
self.password = '******'
self.configs = Config()
self.configs.max_connection_pool_size = 6
self.pool = ConnectionPool()
assert self.pool.init([('127.0.0.1', 3699), ('127.0.0.1', 3700),
('127.0.0.1', 3701)], self.configs)
assert self.pool.connnects() == 0
assert self.pool.in_used_connects() == 0
def test_1_release_by_del(self):
def get_local_session(pool):
session = pool.get_session('root', 'nebula')
assert pool.in_used_connects() == 1
get_local_session(self.pool)
assert self.pool.in_used_connects() == 0
def test_2_reconnect(self):
try:
session = self.pool.get_session('root', 'nebula')
for i in range(0, 5):
if i == 3:
os.system('docker stop nebula-docker-compose_graphd_1')
os.system('docker stop nebula-docker-compose_graphd1_1')
time.sleep(3)
resp = session.execute('SHOW SPACES')
if i >= 3:
assert resp.error_code() == ErrorCode.E_SESSION_INVALID
else:
assert resp.is_succeeded()
time.sleep(2)
session.release()
new_session = self.pool.get_session('root', 'nebula')
new_session.execute('SHOW SPACES')
except Exception as e:
assert False, e
finally:
os.system('docker start nebula-docker-compose_graphd_1')
os.system('docker start nebula-docker-compose_graphd1_1')
time.sleep(5)
for secret_id in (user_secret_id, password_secret_id))
return DEFAULT_NG_CREDENTIAL
def parse_nebula_graphd_endpoint():
ng_endpoints_str = os.environ.get('NG_ENDPOINTS',
'127.0.0.1:9669,').split(",")
ng_endpoints = []
for endpoint in ng_endpoints_str:
if endpoint:
parts = endpoint.split(":") # we dont consider IPv6 now
ng_endpoints.append((parts[0], int(parts[1])))
return ng_endpoints
ng_config = Config()
ng_config.max_connection_pool_size = int(
os.environ.get('NG_MAX_CONN_POOL_SIZE', 10))
ng_endpoints = parse_nebula_graphd_endpoint()
connection_pool = ConnectionPool()
if __name__ == "__main__":
connection_pool.init(ng_endpoints, ng_config)
try:
app.run(host="0.0.0.0", port=5000)
finally:
connection_pool.close()
else:
connection_pool.init(ng_endpoints, ng_config)
class NebulaTestSuite(object):
@classmethod
def set_delay(self):
self.delay = get_delay_time(self.client)
@classmethod
def setup_class(self):
self.spaces = []
address = pytest.cmdline.address.split(':')
self.host = address[0]
self.port = address[1]
self.user = pytest.cmdline.user
self.password = pytest.cmdline.password
self.replica_factor = pytest.cmdline.replica_factor
self.partition_num = pytest.cmdline.partition_num
self.check_format_str = 'result: {}, expect: {}'
self.data_dir = pytest.cmdline.data_dir
self.data_loaded = False
self.create_nebula_clients()
self.set_delay()
self.prepare()
@classmethod
def load_data(self):
self.data_loaded = True
pathlist = Path(self.data_dir).rglob('*.ngql')
for path in pathlist:
print("open: ", path)
with open(path, 'r') as data_file:
space_name = path.name.split(
'.')[0] + datetime.datetime.now().strftime('%H_%M_%S_%f')
self.spaces.append(space_name)
resp = self.execute(
'CREATE SPACE IF NOT EXISTS {space_name}(partition_num={partition_num}, '
'replica_factor={replica_factor}, vid_type=FIXED_STRING(30)); USE {space_name};'
.format(partition_num=self.partition_num,
replica_factor=self.replica_factor,
space_name=space_name))
self.check_resp_succeeded(resp)
lines = data_file.readlines()
ddl = False
ngql_statement = ""
for line in lines:
strip_line = line.strip()
if len(strip_line) == 0:
continue
elif strip_line.startswith('--'):
comment = strip_line[2:]
if comment == 'DDL':
ddl = True
elif comment == 'END':
if ddl:
time.sleep(self.delay)
ddl = False
else:
line = line.rstrip()
ngql_statement += " " + line
if line.endswith(';'):
resp = self.execute(ngql_statement)
self.check_resp_succeeded(resp)
ngql_statement = ""
@classmethod
def drop_data(self):
if self.data_loaded:
drop_stmt = []
for space in self.spaces:
drop_stmt.append('DROP SPACE {}'.format(space))
resp = self.execute(';'.join(drop_stmt))
self.check_resp_succeeded(resp)
@classmethod
def use_nba(self):
resp = self.execute('USE nba;')
self.check_resp_succeeded(resp)
@classmethod
def use_student_space(self):
resp = self.execute('USE student_space;')
self.check_resp_succeeded(resp)
@classmethod
def create_nebula_clients(self):
config = Config()
config.max_connection_pool_size = 20
config.timeout = 60000
# init connection pool
self.client_pool = ConnectionPool()
assert self.client_pool.init([(self.host, self.port)], config)
# get session from the pool
self.client = self.client_pool.get_session(self.user, self.password)
@classmethod
def spawn_nebula_client(self, user, password):
return self.client_pool.get_session(user, password)
@classmethod
def release_nebula_client(self, client):
client.release()
@classmethod
def close_nebula_clients(self):
self.client_pool.close()
@classmethod
def teardown_class(self):
if self.client is not None:
self.cleanup()
self.drop_data()
self.client.release()
self.close_nebula_clients()
@classmethod
def execute(self, ngql, profile=True):
return self.client.execute(
'PROFILE {{{}}}'.format(ngql) if profile else ngql)
@classmethod
def execute(self, ngql, profile=True):
return self.client.execute(
'PROFILE {{{}}}'.format(ngql) if profile else ngql)
@classmethod
def check_rows_with_header(cls, stmt: str, expected: dict):
resp = cls.execute(stmt)
cls.check_resp_succeeded(resp)
if "column_names" in expected:
cls.check_column_names(resp, expected['column_names'])
if "rows" in expected:
cls.check_out_of_order_result(resp, expected['rows'])
@classmethod
def prepare(cls):
if hasattr(cls.prepare, 'is_overridden'):
cls.prepare()
@classmethod
def cleanup(cls):
if hasattr(cls.cleanup, 'is_overridden'):
cls.cleanup()
@classmethod
def check_resp_succeeded(self, resp):
assert (resp.is_succeeded or resp.error_code()
== ttypes.ErrorCode.E_STATEMENT_EMTPY), resp.error_msg()
@classmethod
def check_resp_failed(
self,
resp,
error_code: ttypes.ErrorCode = ttypes.ErrorCode.SUCCEEDED):
if error_code == ttypes.ErrorCode.SUCCEEDED:
assert resp.error_code() != error_code, '{} == {}, {}'.format(
ttypes.ErrorCode._VALUES_TO_NAMES[resp.error_code],
ttypes.ErrorCode._VALUES_TO_NAMES[error_code],
resp.error_msg())
else:
assert resp.error_code() == error_code, '{} != {}, {}'.format(
ttypes.ErrorCode._VALUES_TO_NAMES[resp.error_code],
ttypes.ErrorCode._VALUES_TO_NAMES[error_code],
resp.error_msg())
@classmethod
def search_result(self, resp, expect, is_regex=False):
self.search(resp, expect, is_regex)
@classmethod
def search_not_exist(self, resp, expect, is_regex=False):
self.search(resp, expect, is_regex, False)
@classmethod
def search(self, resp, expect, is_regex=False, exist=True):
if resp.is_empty() and len(expect) == 0:
return
assert not resp.is_empty(), 'resp.data is None'
rows = resp.rows()
assert len(rows) >= len(expect), f'{len(rows)} < {len(expect)}'
new_expect = expect
if not is_regex:
# convert expect to thrift value
new_expect = self.convert_expect(expect)
msg = 'Returned row from nebula could not be found, row: {}, resp: {}'
for exp in new_expect:
values, exp_str = (exp,
str(exp)) if is_regex else (exp.values,
row_to_string(exp))
assert find_in_rows(values, rows) == exist, \
msg.format(exp_str, value_to_string(rows))
@classmethod
def check_column_names(self, resp, expect):
column_names = resp.keys()
assert len(column_names) == len(expect), \
f'Column names does not match, expected: {expect}, actual: {column_names}'
for i in range(len(expect)):
result = column_names[i]
assert expect[i] == result, \
f"different column name, expect: {expect[i]} vs. result: {result}"
@classmethod
def convert_expect(self, expect):
result = []
for row in expect:
assert type(row) is list, f'{str(row)} is not list type'
new_row = CommonTtypes.Row()
new_row.values = list(map(to_value, row))
result.append(new_row)
return result
@classmethod
def check_result(self,
resp,
expect,
ignore_col: Set[int] = set(),
is_regex=False):
if resp.is_empty() and len(expect) == 0:
return
assert not resp.is_empty(), 'resp.data is None'
rows = resp.rows()
assert len(rows) == len(expect), f'{len(rows)}!={len(expect)}'
new_expect = expect
if not is_regex:
# convert expect to thrift value
new_expect = self.convert_expect(expect)
for row, i in zip(rows, range(0, len(new_expect))):
columns = new_expect[i].values if isinstance(
new_expect[i], CommonTtypes.Row) else new_expect[i]
assert len(row.values) - len(ignore_col) == len(columns)
ignored_col_count = 0
for j, col in enumerate(row.values):
if j in ignore_col:
ignored_col_count += 1
continue
exp_val = columns[j - ignored_col_count]
expect_to_string = row_to_string(columns)
assert compare_value(col, exp_val), \
'The returned row from nebula could not be found, row: {}, expect: {}'.format(
row_to_string(row), expect_to_string)
@classmethod
def check_out_of_order_result(self,
resp,
expect,
ignore_col: Set[int] = set()):
if resp.is_empty() and len(expect) == 0:
return
assert not resp.is_empty()
# convert expect to thrift value
new_expect = self.convert_expect(expect)
rows = resp.rows()
sorted_rows = sorted(rows, key=row_to_string)
resp._resp.data.rows = sorted_rows
sorted_expect = sorted(new_expect, key=row_to_string)
# has convert the expect, so set is_regex to True
self.check_result(resp, sorted_expect, ignore_col, True)
@classmethod
def check_empty_result(self, resp):
msg = 'the row was not empty {}'.format(resp)
assert resp.is_empty(), msg
@classmethod
def check_path_result_without_prop(self, rows, expect):
msg = 'len(rows)[%d] != len(expect)[%d]' % (len(rows), len(expect))
assert len(rows) == len(expect), msg
for exp in expect:
path = CommonTtypes.Path()
path.steps = []
for col, j in zip(exp, range(len(exp))):
if j == 0:
src = CommonTtypes.Vertex()
src.vid = bytes(col, encoding='utf-8')
src.tags = []
path.src = src
else:
assert len(col) == 3, \
"{} invalid values size in expect result".format(exp.__repr__())
step = CommonTtypes.Step()
step.name = bytes(col[0], encoding='utf-8')
step.ranking = col[1]
step.type = 1
dst = CommonTtypes.Vertex()
dst.vid = bytes(col[2], encoding='utf-8')
dst.tags = []
step.dst = dst
step.props = {}
path.steps.append(step)
find = False
for row in rows:
assert len(row.values) == 1, \
"invalid values size in rows: {}".format(row)
assert row.values[0].getType() == CommonTtypes.Value.PVAL, \
"invalid column path type: {}".format(row.values[0].getType()())
if row.values[0].get_pVal() == path:
find = True
break
msg = self.check_format_str.format(row.values[0].get_pVal(), path)
assert find, msg
rows.remove(row)
assert len(rows) == 0
@classmethod
def check_error_msg(self, resp, expect):
self.check_resp_failed(resp)
msg = self.check_format_str.format(resp.error_msg(), expect)
err_msg = resp.error_msg()
if isinstance(expect, Pattern):
assert expect.match(err_msg), msg
else:
assert err_msg == expect, msg
@classmethod
def check_exec_plan(cls, resp, expect):
cls.check_resp_succeeded(resp)
if resp.plan_desc() is None:
return
cls.diff_plan_node(resp.plan_desc(), 0, expect, 0)
@classmethod
def diff_plan_node(cls, plan_desc, line_num, expect, expect_idx):
plan_node_desc = plan_desc.plan_node_descs[line_num]
expect_node = expect[expect_idx]
name = bytes.decode(plan_node_desc.name)
assert name.lower().startswith(expect_node[0].lower()), \
"Different plan node: {} vs. {}".format(name, expect_node[0])
if len(expect_node) > 2:
descs = {
bytes.decode(pair.value)
for pair in plan_node_desc.description
}
assert set(expect_node[2]).issubset(descs), \
'Invalid descriptions, expect: {} vs. resp: {}'.format(
'; '.join(map(str, expect_node[2])),
'; '.join(map(str, descs)))
if plan_node_desc.dependencies is None:
return
assert len(expect_node[1]) == len(plan_node_desc.dependencies), \
"Different plan node dependencies: {} vs. {}".format(
len(plan_node_desc.dependencies), len(expect_node[1]))
for i in range(len(plan_node_desc.dependencies)):
line_num = plan_desc.node_index_map[plan_node_desc.dependencies[i]]
cls.diff_plan_node(plan_desc, line_num, expect, expect_node[1][i])
import sys
import time
sys.path.insert(0, '../')
from nebula2.gclient.net import ConnectionPool
from nebula2.Config import Config
from FormatResp import print_resp
if __name__ == '__main__':
client = None
try:
config = Config()
config.max_connection_pool_size = 2
# init connection pool
connection_pool = ConnectionPool()
assert connection_pool.init([('127.0.0.1', 3700), ('127.0.0.1', 3699)],
config)
# get session from the pool
client = connection_pool.get_session('root', 'nebula')
assert client is not None
client.execute(
'CREATE SPACE IF NOT EXISTS test; USE test;'
'CREATE TAG IF NOT EXISTS person(name string, age int);')
# insert data need to sleep after create schema
time.sleep(6)
# insert vertex
class GlobalDataLoader(object):
def __init__(self, data_dir, host, port, user, password):
self.data_dir = data_dir
self.host = host
self.port = port
self.user = user
self.password = password
self.has_load_data = False
def __enter__(self):
config = Config()
config.max_connection_pool_size = 2
config.timeout = 0
# init connection pool
self.client_pool = ConnectionPool()
assert self.client_pool.init([(self.host, self.port)], config)
# get session from the pool
self.client = self.client_pool.get_session(self.user, self.password)
return self
def __exit__(self, exc_type, exc_value, traceback):
if self.has_load_data:
self.drop_data()
self.client.release()
self.client_pool.close()
def load_all_test_data(self):
if self.client is None:
assert False, 'Connect to {}:{}'.format(self.host, self.port)
self.load_nba()
self.load_student()
self.has_load_data = True
# The whole test will load once, for the only read tests
def load_nba(self):
nba_file = self.data_dir + '/data/nba.ngql'
print("open: ", nba_file)
with open(nba_file, 'r') as data_file:
resp = self.client.execute(
'CREATE SPACE IF NOT EXISTS nba(partition_num=10, replica_factor=1, vid_type = fixed_string(30));USE nba;'
)
assert resp.is_succeeded(), resp.error_msg()
lines = data_file.readlines()
ddl = False
ngql_statement = ""
for line in lines:
strip_line = line.strip()
if len(strip_line) == 0:
continue
elif strip_line.startswith('--'):
comment = strip_line[2:]
if comment == 'DDL':
ddl = True
elif comment == 'END':
if ddl:
time.sleep(get_delay_time(self.client))
ddl = False
else:
line = line.rstrip()
ngql_statement += " " + line
if line.endswith(';'):
resp = self.client.execute(ngql_statement)
assert resp.is_succeeded(), resp.error_msg()
ngql_statement = ""
# The whole test will load once, for the only read tests
def load_student(self):
resp = self.client.execute(
'CREATE SPACE IF NOT EXISTS student(partition_num=10, replica_factor=1, vid_type = fixed_string(8)); USE student;'
)
assert resp.is_succeeded(), resp.error_msg()
resp = self.client.execute(
'CREATE TAG IF NOT EXISTS person(name string, age int, gender string);'
)
assert resp.is_succeeded(), resp.error_msg()
resp = self.client.execute(
'CREATE TAG IF NOT EXISTS teacher(grade int, subject string);')
assert resp.is_succeeded(), resp.error_msg()
resp = self.client.execute(
'CREATE TAG IF NOT EXISTS student(grade int, hobby string DEFAULT "");'
)
assert resp.is_succeeded(), resp.error_msg()
resp = self.client.execute(
'CREATE EDGE IF NOT EXISTS is_schoolmate(start_year int, end_year int);'
)
assert resp.is_succeeded(), resp.error_msg()
resp = self.client.execute(
'CREATE EDGE IF NOT EXISTS is_teacher(start_year int, end_year int);'
)
assert resp.is_succeeded(), resp.error_msg()
resp = self.client.execute(
'CREATE EDGE IF NOT EXISTS is_friend(start_year int, intimacy double);'
)
assert resp.is_succeeded(), resp.error_msg()
resp = self.client.execute(
'CREATE EDGE IF NOT EXISTS is_colleagues(start_year int, end_year int);'
)
assert resp.is_succeeded(), resp.error_msg()
# TODO: update the time when config can use
time.sleep(get_delay_time(self.client))
resp = self.client.execute(
'INSERT VERTEX person(name, age, gender), teacher(grade, subject) VALUES \
"2001":("Mary", 25, "female", 5, "Math"), \
"2002":("Ann", 23, "female", 3, "English"), \
"2003":("Julie", 33, "female", 6, "Math"), \
"2004":("Kim", 30,"male", 5, "English"), \
"2005":("Ellen", 27, "male", 4, "Art"), \
"2006":("ZhangKai", 27, "male", 3, "Chinese"), \
"2007":("Emma", 26, "female", 2, "Science"), \
"2008":("Ben", 24, "male", 4, "Music"), \
"2009":("Helen", 24, "male", 2, "Sports") ,\
"2010":("Lilan", 32, "male", 5, "Chinese");')
assert resp.is_succeeded(), resp.error_msg()
resp = self.client.execute(
'INSERT VERTEX person(name, age, gender), student(grade) VALUES \
"1001":("Anne", 7, "female", 2), \
"1002":("Cynthia", 7, "female", 2), \
"1003":("Jane", 6, "male", 2), \
"1004":("Lisa", 8, "female", 3), \
"1005":("Peggy", 8, "male", 3), \
"1006":("Kevin", 9, "male", 3), \
"1007":("WangLe", 8, "male", 3), \
"1008":("WuXiao", 9, "male", 4), \
"1009":("Sandy", 9, "female", 4), \
"1010":("Harry", 9, "female", 4), \
"1011":("Ada", 8, "female", 4), \
"1012":("Lynn", 9, "female", 5), \
"1013":("Bonnie", 10, "female", 5), \
"1014":("Peter", 10, "male", 5), \
"1015":("Carl", 10, "female", 5), \
"1016":("Sonya", 11, "male", 6), \
"1017":("HeNa", 11, "female", 6), \
"1018":("Tom", 12, "male", 6), \
"1019":("XiaMei", 11, "female", 6), \
"1020":("Lily", 10, "female", 6);')
assert resp.is_succeeded(), resp.error_msg()
resp = self.client.execute(
'INSERT EDGE is_schoolmate(start_year, end_year) VALUES \
"1001" -> "1002":(2018, 2019), \
"1001" -> "1003":(2017, 2019), \
"1002" -> "1003":(2017, 2018), \
"1002" -> "1001":(2018, 2019), \
"1004" -> "1005":(2016, 2019), \
"1004" -> "1006":(2017, 2019), \
"1004" -> "1007":(2016, 2018), \
"1005" -> "1004":(2017, 2018), \
"1005" -> "1007":(2017, 2018), \
"1006" -> "1004":(2017, 2018), \
"1006" -> "1007":(2018, 2019), \
"1008" -> "1009":(2015, 2019), \
"1008" -> "1010":(2017, 2019), \
"1008" -> "1011":(2018, 2019), \
"1010" -> "1008":(2017, 2018), \
"1011" -> "1008":(2018, 2019), \
"1012" -> "1013":(2015, 2019), \
"1012" -> "1014":(2017, 2019), \
"1012" -> "1015":(2018, 2019), \
"1013" -> "1012":(2017, 2018), \
"1014" -> "1015":(2018, 2019), \
"1016" -> "1017":(2015, 2019), \
"1016" -> "1018":(2014, 2019), \
"1018" -> "1019":(2018, 2019), \
"1017" -> "1020":(2013, 2018), \
"1017" -> "1016":(2018, 2019);')
assert resp.is_succeeded(), resp.error_msg()
resp = self.client.execute(
'INSERT EDGE is_friend(start_year, intimacy) VALUES \
"1003" -> "1004":(2017, 80.0), \
"1013" -> "1007":(2018, 80.0), \
"1016" -> "1008":(2015, 80.0), \
"1016" -> "1018":(2014, 85.0), \
"1017" -> "1020":(2018, 78.0), \
"1018" -> "1016":(2013, 83.0), \
"1018" -> "1020":(2018, 88.0);')
assert resp.is_succeeded(), resp.error_msg()
resp = self.client.execute(
'INSERT EDGE is_colleagues(start_year, end_year) VALUES \
"2001" -> "2002":(2015, 0), \
"2001" -> "2007":(2014, 0), \
"2001" -> "2003":(2018, 0), \
"2003" -> "2004":(2013, 2017), \
"2002" -> "2001":(2016, 2017), \
"2007" -> "2001":(2013, 2018), \
"2010" -> "2008":(2018, 0);')
assert resp.is_succeeded(), resp.error_msg()
resp = self.client.execute(
'INSERT EDGE is_teacher(start_year, end_year) VALUES \
"2002" -> "1004":(2018, 2019), \
"2002" -> "1005":(2018, 2019), \
"2002" -> "1006":(2018, 2019), \
"2002" -> "1007":(2018, 2019), \
"2002" -> "1009":(2017, 2018), \
"2002" -> "1012":(2015, 2016), \
"2002" -> "1013":(2015, 2016), \
"2002" -> "1014":(2015, 2016), \
"2002" -> "1019":(2014, 2015), \
"2010" -> "1016":(2018,2019), \
"2006" -> "1008":(2017, 2018);')
assert resp.is_succeeded(), resp.error_msg()
def drop_data(self):
resp = self.client.execute('DROP SPACE nba; DROP SPACE student;')
assert resp.is_succeeded(), resp.error_msg()
def setUpClass(cls) -> None:
super().setUpClass()
configs = Config()
configs.max_connection_pool_size = 1
cls.pool = ConnectionPool()
cls.pool.init([('127.0.0.1', 9671)], configs)
cls.session = cls.pool.get_session('root', 'nebula')
resp = cls.session.execute(
'''
CREATE SPACE IF NOT EXISTS test_data(vid_type=FIXED_STRING(8));
USE test_data;
CREATE TAG IF NOT EXISTS person(name string, age int8, grade int16,
friends int32, book_num int64, birthday datetime,
start_school date, morning time, property double,
is_girl bool, child_name fixed_string(10), expend float,
first_out_city timestamp, hobby string);
CREATE TAG IF NOT EXISTS student(name string);
CREATE EDGE IF NOT EXISTS like(likeness double);
CREATE EDGE IF NOT EXISTS friend(start_year int, end_year int);
CREATE TAG INDEX IF NOT EXISTS person_name_index ON person(name(8));
'''
)
assert resp.is_succeeded(), resp.error_msg()
time.sleep(5)
resp = cls.session.execute(
"INSERT VERTEX person(name, age, grade,friends, book_num,"
"birthday, start_school, morning, property,"
"is_girl, child_name, expend, first_out_city) VALUES"
"'Bob':('Bob', 10, 3, 10, 100, datetime('2010-09-10T10:08:02'),"
"date('2017-09-10'), time('07:10:00'), "
"1000.0, false, 'Hello World!', 100.0, 1111),"
"'Lily':('Lily', 9, 3, 10, 100, datetime('2010-09-10T10:08:02'), "
"date('2017-09-10'), time('07:10:00'), "
"1000.0, false, 'Hello World!', 100.0, 1111),"
"'Tom':('Tom', 10, 3, 10, 100, datetime('2010-09-10T10:08:02'), "
"date('2017-09-10'), time('07:10:00'), "
"1000.0, false, 'Hello World!', 100.0, 1111),"
"'Jerry':('Jerry', 9, 3, 10, 100, datetime('2010-09-10T10:08:02'),"
"date('2017-09-10'), time('07:10:00'), "
"1000.0, false, 'Hello World!', 100.0, 1111), "
"'John':('John', 10, 3, 10, 100, datetime('2010-09-10T10:08:02'), "
"date('2017-09-10'), time('07:10:00'), "
"1000.0, false, 'Hello World!', 100.0, 1111)"
)
assert resp.is_succeeded(), resp.error_msg()
resp = cls.session.execute(
"INSERT VERTEX student(name) VALUES "
"'Bob':('Bob'), 'Lily':('Lily'), "
"'Tom':('Tom'), 'Jerry':('Jerry'), 'John':('John')")
assert resp.is_succeeded(), resp.error_msg()
resp = cls.session.execute(
"INSERT EDGE like(likeness) VALUES "
"'Bob'->'Lily':(80.0), "
"'Bob'->'Tom':(70.0), "
"'Jerry'->'Lily':(84.0),"
"'Tom'->'Jerry':(68.3), "
"'Bob'->'John':(97.2)")
assert resp.is_succeeded(), resp.error_msg()
resp = cls.session.execute(
"INSERT EDGE friend(start_year, end_year) VALUES "
"'Bob'->'Lily':(2018, 2020), "
"'Bob'->'Tom':(2018, 2020), "
"'Jerry'->'Lily':(2018, 2020),"
"'Tom'->'Jerry':(2018, 2020), "
"'Bob'->'John':(2018, 2020)")
assert resp.is_succeeded(), resp.error_msg()
class TestSession(TestCase):
@classmethod
def setup_class(self):
self.user_name = 'root'
self.password = '******'
self.configs = Config()
self.configs.max_connection_pool_size = 6
self.pool = ConnectionPool()
assert self.pool.init([('127.0.0.1', 9669), ('127.0.0.1', 9670),
('127.0.0.1', 9671)], self.configs)
assert self.pool.connnects() == 0
assert self.pool.in_used_connects() == 0
def test_1_release_by_del(self):
def get_local_session(pool):
session = pool.get_session('root', 'nebula')
assert pool.in_used_connects() == 1
get_local_session(self.pool)
assert self.pool.in_used_connects() == 0
def test_2_reconnect(self):
try:
session = self.pool.get_session('root', 'nebula')
session.execute(
'CREATE SPACE IF NOT EXISTS test_session(vid_type=FIXED_STRING(8)); USE test_session;'
)
for i in range(0, 5):
if i == 3:
os.system('docker stop nebula-docker-compose_graphd0_1')
os.system('docker stop nebula-docker-compose_graphd1_1')
time.sleep(3)
# the session update later, the expect test
# resp = session.execute('SHOW TAGS')
resp = session.execute('SHOW HOSTS')
assert resp.is_succeeded(), resp.error_msg()
assert resp.space_name() == 'test_session'
time.sleep(2)
session.release()
new_session = self.pool.get_session('root', 'nebula')
new_session.execute('SHOW SPACES')
except Exception as e:
assert False, e
finally:
os.system('docker start nebula-docker-compose_graphd0_1')
os.system('docker start nebula-docker-compose_graphd1_1')
time.sleep(5)
def test_3_session_context(self):
in_used_connects = self.pool.in_used_connects()
with self.pool.session_context('root', 'nebula') as session:
assert self.pool.in_used_connects() == in_used_connects + 1
assert self.pool.in_used_connects() == in_used_connects
def test_4_timeout(self):
try:
configs = Config()
configs.timeout = 100
configs.max_connection_pool_size = 1
pool = ConnectionPool()
assert pool.init([('127.0.0.1', 9669)], configs)
session = pool.get_session(self.user_name, self.password)
ngql = ''
for n in range(0, 500):
ngql = ngql + 'show hosts;'
session.execute(ngql)
assert False, 'expect to get exception'
except Exception as ex:
assert str(ex).find('timed out') > 0
assert True, ex
except Exception as x:
print(x)
import traceback
print(traceback.format_exc())
finally:
if client is not None:
client.release()
if __name__ == '__main__':
config = Config()
config.max_connection_pool_size = 4
# init connection pool
connection_pool = ConnectionPool()
assert connection_pool.init([('127.0.0.1', 9669), ('127.0.0.1', 9670)], config)
# Use multi thread and reuse the session three times
for count in range(0, 3):
threads = list()
for i in range(0, 4):
threads.append(threading.Thread(target=main_test, name='thread{}'.format(i)))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
# close connect pool
from nebula2.gclient.net import ConnectionPool
from nebula2.Config import Config
# define a config
config = Config()
config.max_connection_pool_size = 10
# init connection pool
connection_pool = ConnectionPool()
# if the given servers are ok, return true, else return false
ok = connection_pool.init([('192.168.71.129', 9669)], config)
def runGql(nGQL):
print(nGQL)
# get session from the pool
session = connection_pool.get_session('root', 'nebula')
# show hosts
result = session.execute(nGQL)
# release session
session.release()
res = result._resp.__dict__
return res
class TestConnectionPool(TestCase):
@classmethod
def setup_class(self):
self.addresses = list()
self.addresses.append(('127.0.0.1', 9669))
self.addresses.append(('127.0.0.1', 9670))
self.configs = Config()
self.configs.min_connection_pool_size = 2
self.configs.max_connection_pool_size = 4
self.configs.idle_time = 2000
self.configs.interval_check = 2
self.pool = ConnectionPool()
assert self.pool.init(self.addresses, self.configs)
assert self.pool.connnects() == 2
def test_right_hostname(self):
pool = ConnectionPool()
assert pool.init([('localhost', 9669)], Config())
def test_wrong_hostname(self):
pool = ConnectionPool()
try:
pool.init([('wrong_host', 9669)], Config())
assert False
except InValidHostname:
assert True
def test_ping(self):
assert self.pool.ping(('127.0.0.1', 9669))
assert self.pool.ping(('127.0.0.1', 5000)) is False
def test_init_failed(self):
# init succeeded
pool1 = ConnectionPool()
addresses = list()
addresses.append(('127.0.0.1', 9669))
addresses.append(('127.0.0.1', 9670))
assert pool1.init(addresses, Config())
# init failed, connected failed
pool2 = ConnectionPool()
addresses = list()
addresses.append(('127.0.0.1', 3800))
try:
pool2.init(addresses, Config())
assert False
except Exception:
assert True
# init failed, hostname not existed
try:
pool3 = ConnectionPool()
addresses = list()
addresses.append(('not_exist_hostname', 3800))
assert not pool3.init(addresses, Config())
except InValidHostname:
assert True, "We expected get the exception"
def test_get_session(self):
# get session succeeded
sessions = list()
for num in range(0, self.configs.max_connection_pool_size):
session = self.pool.get_session('root', 'nebula')
resp = session.execute('SHOW SPACES')
assert resp.is_succeeded()
sessions.append(session)
# get session failed
try:
self.pool.get_session('root', 'nebula')
except NotValidConnectionException:
assert True
assert self.pool.in_used_connects() == 4
# release session
for session in sessions:
session.release()
assert self.pool.in_used_connects() == 0
assert self.pool.connnects() == 4
# test get session after release
for num in range(0, self.configs.max_connection_pool_size - 1):
session = self.pool.get_session('root', 'nebula')
resp = session.execute('SHOW SPACES')
assert resp.is_succeeded()
sessions.append(session)
assert self.pool.in_used_connects() == 3
assert self.pool.connnects() == 4
# test the idle connection delete
time.sleep(5)
assert self.pool.connnects() == 3
def test_stop_close(self):
session = self.pool.get_session('root', 'nebula')
assert session is not None
resp = session.execute('SHOW SPACES')
assert resp.is_succeeded()
self.pool.close()
try:
new_session = self.pool.get_session('root', 'nebula')
except NotValidConnectionException:
assert True
except Exception as e:
assert False, "We don't expect reach here:{}".format(e)
try:
session.execute('SHOW SPACES')
except IOErrorException:
assert True
except Exception as e:
assert False, "We don't expect reach here:".format(e)
def test_right_hostname(self):
pool = ConnectionPool()
assert pool.init([('localhost', 9669)], Config())
