def test_shard_mapping_list_mappings(self):
shard_mappings = ShardMapping.list("RANGE")
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_1, shard_mappings[0]))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_2, shard_mappings[1]))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_3, shard_mappings[2]))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_4, shard_mappings[3]))
def test_fetch_sharding_scheme(self):
range_sharding_specifications = RangeShardingSpecification.list(1)
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[0],
self.__range_sharding_specification_1))
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[1],
self.__range_sharding_specification_2))
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[2],
self.__range_sharding_specification_3))
def test_fetch_shard_mapping(self):
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_2 = ShardMapping.fetch("db2.t2")
shard_mapping_3 = ShardMapping.fetch("db3.t3")
shard_mapping_4 = ShardMapping.fetch("db4.t4")
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_1, shard_mapping_1))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_2, shard_mapping_2))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_3, shard_mapping_3))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_4, shard_mapping_4))
def test_fetch_sharding_scheme(self):
range_sharding_specifications = RangeShardingSpecification.list(1)
self.assertTrue(
ShardingUtils.compare_range_specifications(
range_sharding_specifications[0],
self.__range_sharding_specification_1))
self.assertTrue(
ShardingUtils.compare_range_specifications(
range_sharding_specifications[1],
self.__range_sharding_specification_2))
self.assertTrue(
ShardingUtils.compare_range_specifications(
range_sharding_specifications[2],
self.__range_sharding_specification_3))
def test_shard_mapping_list_mappings(self):
shard_mappings = ShardMapping.list("RANGE")
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_1,
shard_mappings[0]))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_2,
shard_mappings[1]))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_3,
shard_mappings[2]))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_4,
shard_mappings[3]))
def test_shard_mapping_list_mappings(self):
"""Test the listing of all HASH shards in a shard mapping.
"""
shard_mappings = ShardMapping.list("HASH")
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_1,
shard_mappings[0]))
def test_fetch_shard_mapping(self):
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_2 = ShardMapping.fetch("db2.t2")
shard_mapping_3 = ShardMapping.fetch("db3.t3")
shard_mapping_4 = ShardMapping.fetch("db4.t4")
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_1,
shard_mapping_1))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_2,
shard_mapping_2))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_3,
shard_mapping_3))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_4,
shard_mapping_4))
def hash_sharding_specification_in_list(self, hash_sharding_spec,
hash_sharding_specification_list):
"""Verify if the given hash sharding specification is present in
the list of hash sharding specifications.
:param hash_sharding_spec: The hash sharding specification that
needs to be lookedup.
:param hash_sharding_specification_list: The list of hash sharding
specifications
:return: True if the hash sharding specification is present.
False otherwise.
"""
for i in range(0, len(hash_sharding_specification_list)):
if ShardingUtils.compare_hash_specifications(
hash_sharding_spec, hash_sharding_specification_list[i]):
return True
return False
def hash_sharding_specification_in_list(self,
hash_sharding_spec,
hash_sharding_specification_list
):
"""Verify if the given hash sharding specification is present in
the list of hash sharding specifications.
:param hash_sharding_spec: The hash sharding specification that
needs to be lookedup.
:param hash_sharding_specification_list: The list of hash sharding
specifications
:return: True if the hash sharding specification is present.
False otherwise.
"""
for i in range(0, len(hash_sharding_specification_list)):
if ShardingUtils.compare_hash_specifications(
hash_sharding_spec,
hash_sharding_specification_list[i]
):
return True
return False
def test_shard_split(self):
split_cnt_1 = 0
split_cnt_2 = 0
shard_server_1 = None
shard_server_2 = None
expected_address_list_1 = \
[MySQLInstances().get_address(2), MySQLInstances().get_address(3)]
expected_address_list_2 = \
[MySQLInstances().get_address(4), MySQLInstances().get_address(5)]
status = self.proxy.sharding.split_shard("1", "GROUPID3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
for i in range(1, 100):
status = self.proxy.sharding.lookup_servers("db1.t1", i, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
obtained_uuid_list = [obtained_server_list[0][0],
obtained_server_list[1][0]]
obtained_address_list = [obtained_server_list[0][1],
obtained_server_list[1][1]]
try:
self.assertEqual(
set(expected_address_list_1), set(obtained_address_list)
)
split_cnt_1 = split_cnt_1 + 1
if shard_server_1 is None:
shard_server_1 = MySQLServer.fetch(obtained_uuid_list[0])
except AssertionError:
self.assertEqual(
set(expected_address_list_2), set(obtained_address_list)
)
split_cnt_2 = split_cnt_2 + 1
if shard_server_2 is None:
shard_server_2 = MySQLServer.fetch(obtained_uuid_list[0])
#Ensure that both the splits have been utilized.
self.assertTrue(split_cnt_1 > 0)
self.assertTrue(split_cnt_2 > 0)
shard_server_1.connect()
shard_server_2.connect()
row_cnt_shard_1 = shard_server_1.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_2 = shard_server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
#Ensure that the split has happened, the number of values in
#each shard should be less than the original.
self.assertTrue(int(row_cnt_shard_1[0][0]) < 100)
self.assertTrue(int(row_cnt_shard_2[0][0]) < 100)
#Ensure tha two new shard_ids have been generated.
hash_sharding_specifications = HashShardingSpecification.list(1)
self.assertTrue(ShardingUtils.compare_hash_specifications(
hash_sharding_specifications[1],
HashShardingSpecification.fetch(2)))
self.assertTrue(ShardingUtils.compare_hash_specifications(
hash_sharding_specifications[0],
HashShardingSpecification.fetch(3)))
def test_shard_mapping_list_mappings(self):
"""Test the listing of all HASH shards in a shard mapping.
"""
shard_mappings = ShardMapping.list("HASH")
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_1, shard_mappings[0]))
def test_shard_split(self):
status = self.proxy.sharding.split_shard("1", "GROUPID3", "600")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
for idx in range(0, 2):
server_uuid = obtained_server_list[idx][0]
shard_server = MySQLServer.fetch(server_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT NAME FROM db1.t1",
{"fetch" : True})
self.assertEqual(len(rows), 3)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
self.assertEqual(rows[2][0], 'TEST 3')
status = self.proxy.sharding.lookup_servers("db1.t1", 800, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
for idx in range(0, 2):
server_uuid = obtained_server_list[idx][0]
shard_server = MySQLServer.fetch(server_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT NAME FROM db1.t1",
{"fetch" : True})
self.assertEqual(len(rows), 4)
self.assertEqual(rows[0][0], 'TEST 4')
self.assertEqual(rows[1][0], 'TEST 5')
self.assertEqual(rows[2][0], 'TEST 6')
self.assertEqual(rows[3][0], 'TEST 7')
status = self.proxy.sharding.lookup_servers("1", 500, "GLOBAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
for idx in range(0, 2):
if obtained_server_list[idx][2]:
global_master_uuid = obtained_server_list[idx][0]
break
global_master = MySQLServer.fetch(global_master_uuid)
global_master.connect()
global_master.exec_stmt("DROP DATABASE IF EXISTS global_db")
global_master.exec_stmt("CREATE DATABASE global_db")
global_master.exec_stmt("CREATE TABLE global_db.global_table"
"(userID INT, name VARCHAR(30))")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(101, 'TEST 1')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(202, 'TEST 2')")
status = self.proxy.group.promote("GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_change_to_candidate).")
sleep(5)
status = self.proxy.sharding.lookup_servers("1", 500, "GLOBAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
for idx in range(0, 2):
if obtained_server_list[idx][2]:
global_master_uuid = obtained_server_list[idx][0]
break
global_master = MySQLServer.fetch(global_master_uuid)
global_master.connect()
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(303, 'TEST 3')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(404, 'TEST 4')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(505, 'TEST 5')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(606, 'TEST 6')")
sleep(5)
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
for idx in range(0, 2):
if obtained_server_list[idx][2]:
shard_uuid = obtained_server_list[idx][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT NAME FROM global_db.global_table", {"fetch" : True}
)
self.assertEqual(len(rows), 6)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
self.assertEqual(rows[2][0], 'TEST 3')
self.assertEqual(rows[3][0], 'TEST 4')
self.assertEqual(rows[4][0], 'TEST 5')
self.assertEqual(rows[5][0], 'TEST 6')
#Ensure tha two new shard_ids have been generated.
range_sharding_specifications = RangeShardingSpecification.list(1)
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[0],
RangeShardingSpecification.fetch(2)))
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[1],
RangeShardingSpecification.fetch(3)))
def test_shard_split(self):
status = self.proxy.sharding.split_shard("1", "GROUPID3", "600")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
for info in self.check_xmlrpc_iter(status):
server_uuid = info['server_uuid']
shard_server = fetch_test_server(server_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT NAME FROM db1.t1",
{"fetch" : True})
self.assertEqual(len(rows), 3)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
self.assertEqual(rows[2][0], 'TEST 3')
status = self.proxy.sharding.lookup_servers("db1.t1", 800, "LOCAL")
for info in self.check_xmlrpc_iter(status):
server_uuid = info['server_uuid']
shard_server = fetch_test_server(server_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT NAME FROM db1.t1",
{"fetch" : True})
self.assertEqual(len(rows), 4)
self.assertEqual(rows[0][0], 'TEST 4')
self.assertEqual(rows[1][0], 'TEST 5')
self.assertEqual(rows[2][0], 'TEST 6')
self.assertEqual(rows[3][0], 'TEST 7')
status = self.proxy.sharding.lookup_servers("1", 500, "GLOBAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
global_master = fetch_test_server(info['server_uuid'])
global_master.connect()
global_master.exec_stmt("DROP DATABASE IF EXISTS global_db")
global_master.exec_stmt("CREATE DATABASE global_db")
global_master.exec_stmt("CREATE TABLE global_db.global_table"
"(userID INT, name VARCHAR(30))")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(101, 'TEST 1')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(202, 'TEST 2')")
status = self.proxy.group.promote("GROUPID1")
self.check_xmlrpc_command_result(status)
sleep(5)
status = self.proxy.sharding.lookup_servers("1", 500, "GLOBAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
global_master = fetch_test_server(info['server_uuid'])
global_master.connect()
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(303, 'TEST 3')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(404, 'TEST 4')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(505, 'TEST 5')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(606, 'TEST 6')")
sleep(5)
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT NAME FROM global_db.global_table", {"fetch" : True}
)
self.assertEqual(len(rows), 6)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
self.assertEqual(rows[2][0], 'TEST 3')
self.assertEqual(rows[3][0], 'TEST 4')
self.assertEqual(rows[4][0], 'TEST 5')
self.assertEqual(rows[5][0], 'TEST 6')
#Ensure tha two new shard_ids have been generated.
range_sharding_specifications = RangeShardingSpecification.list(1)
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[0],
RangeShardingSpecification.fetch(2)))
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[1],
RangeShardingSpecification.fetch(3)))
def test_shard_split(self):
split_cnt_1 = 0
split_cnt_2 = 0
shard_server_1 = None
shard_server_2 = None
expected_address_list_1 = \
[MySQLInstances().get_address(2), MySQLInstances().get_address(3)]
expected_address_list_2 = \
[MySQLInstances().get_address(4), MySQLInstances().get_address(5)]
status = self.proxy.sharding.split_shard("1", "GROUPID3")
self.check_xmlrpc_command_result(status)
for i in range(1, 100):
status = self.proxy.sharding.lookup_servers("db1.t1", i, "LOCAL")
obtained_uuid_list = [
info['server_uuid'] for info in self.check_xmlrpc_iter(status)
]
obtained_address_list = [
info['address'] for info in self.check_xmlrpc_iter(status)
]
try:
self.assertEqual(set(expected_address_list_1),
set(obtained_address_list))
split_cnt_1 = split_cnt_1 + 1
if shard_server_1 is None:
shard_server_1 = fetch_test_server(obtained_uuid_list[0])
except AssertionError:
self.assertEqual(set(expected_address_list_2),
set(obtained_address_list))
split_cnt_2 = split_cnt_2 + 1
if shard_server_2 is None:
shard_server_2 = fetch_test_server(obtained_uuid_list[0])
#Ensure that both the splits have been utilized.
self.assertTrue(split_cnt_1 > 0)
self.assertTrue(split_cnt_2 > 0)
shard_server_1.connect()
shard_server_2.connect()
row_cnt_shard_1 = shard_server_1.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_2 = shard_server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
#Ensure that the split has happened, the number of values in
#each shard should be less than the original.
self.assertTrue(int(row_cnt_shard_1[0][0]) < 100)
self.assertTrue(int(row_cnt_shard_2[0][0]) < 100)
#Ensure tha two new shard_ids have been generated.
hash_sharding_specifications = HashShardingSpecification.list(1)
self.assertTrue(
ShardingUtils.compare_hash_specifications(
hash_sharding_specifications[1],
HashShardingSpecification.fetch(2)))
self.assertTrue(
ShardingUtils.compare_hash_specifications(
hash_sharding_specifications[0],
HashShardingSpecification.fetch(3)))
