def test_managment(self):
"""Test adding server to a group.
"""
options_1 = {
"uuid": _uuid.UUID("bb75b12b-98d1-414c-96af-9e9d4b179678"),
"address": "server_1.mysql.com:3060",
}
server_1 = MySQLServer(**options_1)
MySQLServer.add(server_1)
options_2 = {
"uuid": _uuid.UUID("aa75a12a-98d1-414c-96af-9e9d4b179678"),
"address": "server_2.mysql.com:3060",
}
server_2 = MySQLServer(**options_2)
MySQLServer.add(server_2)
group_1 = Group("oracle.com", "First description.")
Group.add(group_1)
# Add servers to a group
group_1.add_server(server_1)
group_1.add_server(server_2)
self.assertRaises(AssertionError, group_1.add_server, server_1)
self.assertEqual(len(group_1.servers()), 2)
# Remove servers to a group
group_1.remove_server(server_1)
group_1.remove_server(server_2)
self.assertRaises(AssertionError, group_1.remove_server, server_1)
self.assertEqual(len(group_1.servers()), 0)
def test_managment(self):
"""Test adding server to a group.
"""
options_1 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_1.mysql.com:3060",
}
server_1 = MySQLServer(**options_1)
MySQLServer.add(server_1)
options_2 = {
"uuid" : _uuid.UUID("{aa75a12a-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_2.mysql.com:3060",
}
server_2 = MySQLServer(**options_2)
MySQLServer.add(server_2)
group_1 = Group("oracle.com", "First description.")
Group.add(group_1)
# Add servers to a group
group_1.add_server(server_1)
group_1.add_server(server_2)
self.assertRaises(AssertionError, group_1.add_server, server_1)
self.assertEqual(len(group_1.servers()), 2)
# Remove servers to a group
group_1.remove_server(server_1)
group_1.remove_server(server_2)
self.assertRaises(AssertionError, group_1.remove_server, server_1)
self.assertEqual(len(group_1.servers()), 0)
class TestShardingServices(tests.utils.TestCase):
def setUp(self):
"""Configure the existing environment
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(3),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add(self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__options_5 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(4),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add(self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__options_6 = {
"uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(5),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add(self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=2)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=3)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(2, "db2.t2", "userID2")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(3, "db3.t3", "userID3")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(1, "GROUPID2,GROUPID3",
"ENABLED")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(2, "GROUPID4/0,GROUPID5/101",
"ENABLED")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(3, "GROUPID6", "ENABLED")
self.check_xmlrpc_command_result(status)
def test_list_shard_defns(self):
expected_shard_mapping_list1 = tests.utils.make_mapping_result(
[[1, "HASH", "GROUPID1"], [2, "RANGE", "GROUPID1"],
[3, "HASH", "GROUPID1"]])
status = self.proxy.sharding.list_definitions()
self.check_xmlrpc_result(status, expected_shard_mapping_list1)
def test_list_shard_mappings(self):
expected_hash = tests.utils.make_shard_mapping_list_result([
[1, 'HASH', "db1.t1", "GROUPID1", "userID1"],
[3, 'HASH', "db3.t3", "GROUPID1", "userID3"],
])
status = self.proxy.sharding.list_tables("HASH")
self.check_xmlrpc_result(status, expected_hash)
expected_range = tests.utils.make_shard_mapping_list_result([
[2, "RANGE", "db2.t2", "GROUPID1", "userID2"],
])
status = self.proxy.sharding.list_tables("RANGE")
self.check_xmlrpc_result(status, expected_range)
def test_shard_prune(self):
status = self.proxy.sharding.prune_shard("db2.t2")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db2.t2", 1, "LOCAL")
info = self.check_xmlrpc_simple(status, {})
shard_uuid = info['server_uuid']
shard_server = fetch_test_server(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db2.t2",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt("SELECT MAX(userID2) FROM db2.t2",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt("SELECT MIN(userID2) FROM db2.t2",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 1)
status = self.proxy.sharding.lookup_servers("db2.t2", 101, "LOCAL")
info = self.check_xmlrpc_simple(status, {})
shard_uuid = info['server_uuid']
shard_server = fetch_test_server(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db2.t2",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt("SELECT MAX(userID2) FROM db2.t2",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt("SELECT MIN(userID2) FROM db2.t2",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 101)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestSharding(unittest.TestCase):
"""Test dump interface associated to sharding.
"""
def setUp(self):
"""Configure the existing environment
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_1.mysql.com:3060",
}
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa75a12a-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_2.mysql.com:3060",
}
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
self.__group_1.add_server(self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{dd75a12a-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_4.mysql.com:3060",
}
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_3)
self.__group_2.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_3)
self.__options_5 = {
"uuid" : _uuid.UUID("{ee75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{ff75a12a-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_6.mysql.com:3060",
}
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_5)
self.__group_3.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_5)
self.__options_1_host, self.__options_1_port = \
server_utils.split_host_port(self.__options_1["address"], 13001)
self.__options_2_host, self.__options_2_port = \
server_utils.split_host_port(self.__options_2["address"], 13001)
self.__options_3_host, self.__options_3_port = \
server_utils.split_host_port(self.__options_3["address"], 13001)
self.__options_4_host, self.__options_4_port = \
server_utils.split_host_port(self.__options_4["address"], 13001)
self.__options_5_host, self.__options_5_port = \
server_utils.split_host_port(self.__options_5["address"], 13001)
self.__options_6_host, self.__options_6_port = \
server_utils.split_host_port(self.__options_6["address"], 13001)
group_4 = Group("GROUPID4", "4TH description.")
Group.add(group_4)
group_5 = Group("GROUPID5", "5TH description.")
Group.add(group_5)
group_6 = Group("GROUPID6", "6TH description.")
Group.add(group_6)
group_7 = Group("GROUPID7", "7TH description.")
Group.add(group_7)
group_8 = Group("GROUPID8", "8TH description.")
Group.add(group_8)
group_9 = Group("GROUPID9", "9TH description.")
Group.add(group_9)
group_10 = Group("GROUPID10", "10TH description.")
Group.add(group_10)
group_11 = Group("GROUPID11", "11TH description.")
Group.add(group_11)
group_12 = Group("GROUPID12", "12TH description.")
Group.add(group_12)
group_13 = Group("GROUPID13", "13TH description.")
Group.add(group_13)
group_14 = Group("GROUPID14", "14TH description.")
Group.add(group_14)
self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEquals( self.__shard_mapping_list, [])
self.__shard_mapping_id_1 = ShardMapping.define("RANGE", "GROUPID10")
self.__shard_mapping_id_2 = ShardMapping.define("RANGE", "GROUPID11")
self.__shard_mapping_id_3 = ShardMapping.define("RANGE", "GROUPID12")
#Test with sharding type values in lower case
self.__shard_mapping_id_4 = ShardMapping.define("range", "GROUPID13")
self.__shard_mapping_id_5 = ShardMapping.define("range", "GROUPID14")
self.__shard_mapping_1 = \
ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID1")
self.__shard_mapping_2 = \
ShardMapping.add(self.__shard_mapping_id_2, "db2.t2", "userID2")
self.__shard_mapping_3 = \
ShardMapping.add(self.__shard_mapping_id_3, "db3.t3", "userID3")
self.__shard_mapping_4 = \
ShardMapping.add(self.__shard_mapping_id_4, "db4.t4", "userID4")
self.__shard_mapping_5 = \
ShardMapping.add(self.__shard_mapping_id_5, "prune_db.prune_table",
"userID")
self.__shard_id_1 = Shards.add("GROUPID1", "ENABLED")
self.__shard_id_2 = Shards.add("GROUPID10", "ENABLED")
self.__shard_id_3 = Shards.add("GROUPID11", "DISABLED")
self.__shard_id_4 = Shards.add("GROUPID4", "ENABLED")
self.__shard_id_5 = Shards.add("GROUPID5", "ENABLED")
self.__shard_id_6 = Shards.add("GROUPID6", "ENABLED")
self.__shard_id_7 = Shards.add("GROUPID7", "ENABLED")
self.__shard_id_8 = Shards.add("GROUPID8", "ENABLED")
self.__shard_id_9 = Shards.add("GROUPID9", "ENABLED")
self.__shard_id_10 = Shards.add("GROUPID2", "ENABLED")
self.__shard_id_11 = Shards.add("GROUPID3", "ENABLED")
self.__range_sharding_specification_1 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
0,
self.__shard_id_1.shard_id
)
self.__range_sharding_specification_2 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
1001,
self.__shard_id_2.shard_id
)
self.__range_sharding_specification_3 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
2001,
self.__shard_id_3.shard_id
)
self.__range_sharding_specification_4 = RangeShardingSpecification.add(
self.__shard_mapping_2.shard_mapping_id,
3001,
self.__shard_id_4.shard_id
)
self.__range_sharding_specification_5 = RangeShardingSpecification.add(
self.__shard_mapping_2.shard_mapping_id,
4001,
self.__shard_id_5.shard_id
)
self.__range_sharding_specification_6 = RangeShardingSpecification.add(
self.__shard_mapping_3.shard_mapping_id,
6001,
self.__shard_id_6.shard_id
)
self.__range_sharding_specification_7 = RangeShardingSpecification.add(
self.__shard_mapping_3.shard_mapping_id,
7001,
self.__shard_id_7.shard_id
)
self.__range_sharding_specification_8 = RangeShardingSpecification.add(
self.__shard_mapping_4.shard_mapping_id,
8001,
self.__shard_id_8.shard_id
)
self.__range_sharding_specification_9 = RangeShardingSpecification.add(
self.__shard_mapping_4.shard_mapping_id,
10001,
self.__shard_id_9.shard_id
)
self.__range_sharding_specification_10 = RangeShardingSpecification.add(
self.__shard_mapping_5.shard_mapping_id,
100,
self.__shard_id_10.shard_id)
self.__range_sharding_specification_11 = RangeShardingSpecification.add(
self.__shard_mapping_5.shard_mapping_id,
201,
self.__shard_id_11.shard_id)
READ_ONLY = MySQLServer.get_mode_idx(MySQLServer.READ_ONLY)
READ_WRITE = MySQLServer.get_mode_idx(MySQLServer.READ_WRITE)
SECONDARY = MySQLServer.get_status_idx(MySQLServer.SECONDARY)
PRIMARY = MySQLServer.get_status_idx(MySQLServer.PRIMARY)
self.__setofservers = [0, 0, 0,
[[str(self.__server_1.uuid),
'GROUPID1', self.__options_1_host, self.__options_1_port,
READ_ONLY, SECONDARY, 1.0],
[str(self.__server_2.uuid),
'GROUPID1', self.__options_2_host, self.__options_2_port,
READ_ONLY, SECONDARY, 1.0],
[str(self.__server_3.uuid),
'GROUPID2', self.__options_3_host, self.__options_3_port,
READ_WRITE, PRIMARY, 1.0],
[str(self.__server_4.uuid),
'GROUPID2', self.__options_4_host, self.__options_4_port,
READ_ONLY, SECONDARY, 1.0],
[str(self.__server_5.uuid),
'GROUPID3', self.__options_5_host, self.__options_5_port,
READ_WRITE, PRIMARY, 1.0],
[str(self.__server_6.uuid),
'GROUPID3', self.__options_6_host, self.__options_6_port,
READ_ONLY, SECONDARY, 1.0]]]
self.__setofservers_1 = [0, 0, 0,
[[str(self.__server_1.uuid),
'GROUPID1', self.__options_1_host, self.__options_1_port,
READ_ONLY, SECONDARY, 1.0],
[str(self.__server_2.uuid),
'GROUPID1', self.__options_2_host, self.__options_2_port,
READ_ONLY, SECONDARY, 1.0]]]
self.__setofservers_2 = [0, 0, 0,
[[str(self.__server_1.uuid),
'GROUPID1', self.__options_1_host, self.__options_1_port,
READ_ONLY, SECONDARY, 1.0],
[str(self.__server_2.uuid),
'GROUPID1', self.__options_2_host, self.__options_2_port,
READ_ONLY, SECONDARY, 1.0],
[str(self.__server_3.uuid),
'GROUPID2', self.__options_3_host, self.__options_3_port,
READ_WRITE, PRIMARY, 1.0],
[str(self.__server_4.uuid),
'GROUPID2', self.__options_4_host, self.__options_4_port,
READ_ONLY, SECONDARY, 1.0]]]
self.__setofservers_3 = [0, 0, 0,
[[str(self.__server_1.uuid),
'GROUPID1', self.__options_1_host, self.__options_1_port,
READ_ONLY, SECONDARY, 1.0],
[str(self.__server_2.uuid),
'GROUPID1', self.__options_2_host, self.__options_2_port,
READ_ONLY, SECONDARY, 1.0],
[str(self.__server_3.uuid),
'GROUPID2', self.__options_3_host, self.__options_3_port,
READ_WRITE, PRIMARY, 1.0],
[str(self.__server_4.uuid),
'GROUPID2', self.__options_4_host, self.__options_4_port,
READ_ONLY, SECONDARY, 1.0],
[str(self.__server_5.uuid),
'GROUPID3', self.__options_5_host, self.__options_5_port,
READ_WRITE, PRIMARY, 1.0],
[str(self.__server_6.uuid),
'GROUPID3', self.__options_6_host, self.__options_6_port,
READ_ONLY, SECONDARY, 1.0]]]
self.__setoftables = [0, 0, 0, [['db1', 't1', 'userID1', '1'],
['db2', 't2', 'userID2', '2'],
['db3', 't3', 'userID3', '3'],
['db4', 't4', 'userID4', '4'],
['prune_db', 'prune_table', 'userID', '5']]]
self.__setoftables_1 = [0, 0, 0, [['db1', 't1', 'userID1', '1']]]
self.__setoftables_2 = [0, 0, 0, [['db1', 't1', 'userID1', '1'],
['db2', 't2', 'userID2', '2']]]
self.__setoftables_3 = [0, 0, 0, [['db1', 't1', 'userID1', '1'],
['db2', 't2', 'userID2', '2'],
['db3', 't3', 'userID3', '3']]]
self.__setofshardmaps = [0, 0, 0, [['1', 'RANGE', 'GROUPID10'],
['2', 'RANGE', 'GROUPID11'],
['3', 'RANGE', 'GROUPID12'],
['4', 'RANGE', 'GROUPID13'],
['5', 'RANGE', 'GROUPID14']]]
self.__setofshardmaps_1 = [0, 0, 0, [['1', 'RANGE', 'GROUPID10']]]
self.__setofshardmaps_2 = [0, 0, 0, [['1', 'RANGE', 'GROUPID10'],
['2', 'RANGE', 'GROUPID11']]]
self.__setofshardmaps_3 = [0, 0, 0, [['1', 'RANGE', 'GROUPID10'],
['2', 'RANGE', 'GROUPID11'],
['3', 'RANGE', 'GROUPID12']]]
self.__setofshardindexes = [0, 0, 0, [['0', '1', '1', 'GROUPID1'],
['1001', '1', '2', 'GROUPID10'],
['3001', '2', '4', 'GROUPID4'],
['4001', '2', '5', 'GROUPID5'],
['6001', '3', '6', 'GROUPID6'],
['7001', '3', '7', 'GROUPID7'],
['8001', '4', '8', 'GROUPID8'],
['10001', '4', '9', 'GROUPID9'],
['100', '5', '10', 'GROUPID2'],
['201', '5', '11', 'GROUPID3']]]
self.__setofshardindexes_1 = [0, 0, 0, [['0', '1', '1', 'GROUPID1'],
['1001', '1', '2', 'GROUPID10']]]
self.__setofshardindexes_3 = [0, 0, 0, [['0', '1', '1', 'GROUPID1'],
['1001', '1', '2', 'GROUPID10'],
['3001', '2', '4', 'GROUPID4'],
['4001', '2', '5', 'GROUPID5'],
['6001', '3', '6', 'GROUPID6'],
['7001', '3', '7', 'GROUPID7']]]
self.__setofshardindexes_5 = [0, 0, 0, [['0', '1', '1', 'GROUPID1'],
['1001', '1', '2', 'GROUPID10'],
['3001', '2', '4', 'GROUPID4'],
['4001', '2', '5', 'GROUPID5'],
['6001', '3', '6', 'GROUPID6'],
['7001', '3', '7', 'GROUPID7'],
['8001', '4', '8', 'GROUPID8'],
['10001', '4', '9', 'GROUPID9'],
['100', '5', '10', 'GROUPID2'],
['201', '5', '11', 'GROUPID3']]]
self.__shardinginformation_1 = [0, 0, 0, [['db1', 't1', 'userID1', '0',
'1', 'RANGE', 'GROUPID1', 'GROUPID10'],
['db1', 't1', 'userID1', '1001',
'2', 'RANGE', 'GROUPID10', 'GROUPID10']]]
self.__shardinginformation_2 = [0, 0, 0, [['db1', 't1', 'userID1', '0',
'1', 'RANGE', 'GROUPID1', 'GROUPID10'],
['db1', 't1', 'userID1', '1001',
'2', 'RANGE', 'GROUPID10', 'GROUPID10'],
['db2', 't2', 'userID2', '3001',
'4', 'RANGE', 'GROUPID4', 'GROUPID11'],
['db2', 't2', 'userID2', '4001',
'5', 'RANGE', 'GROUPID5', 'GROUPID11']]]
self.__shardinginformation_3 = [0, 0, 0, [['db1', 't1', 'userID1', '0',
'1', 'RANGE', 'GROUPID1', 'GROUPID10'],
['db1', 't1', 'userID1', '1001',
'2', 'RANGE', 'GROUPID10', 'GROUPID10'],
['db2', 't2', 'userID2', '3001',
'4','RANGE', 'GROUPID4', 'GROUPID11'],
['db2', 't2', 'userID2', '4001',
'5', 'RANGE', 'GROUPID5', 'GROUPID11'],
['db3', 't3', 'userID3', '6001',
'6', 'RANGE', 'GROUPID6', 'GROUPID12'],
['db3', 't3', 'userID3', '7001',
'7', 'RANGE', 'GROUPID7', 'GROUPID12']]]
def test_dumps(self):
"""Test dump interface associated to sharding.
"""
self.assertEqual(self.__setofservers,
self.proxy.dump.servers(0))
self.assertEqual(self.__setofservers_1,
self.proxy.dump.servers(0, "GROUPID1"))
self.assertEqual(self.__setofservers_2,
self.proxy.dump.servers(0, "GROUPID1,GROUPID2"))
self.assertEqual(self.__setofservers_3,
self.proxy.dump.servers(0, "GROUPID1,GROUPID2,GROUPID3"))
self.assertEqual(self.proxy.dump.shard_tables(0),
self.__setoftables)
self.assertEqual(self.proxy.dump.shard_tables(0, "1"),
self.__setoftables_1)
self.assertEqual(self.proxy.dump.shard_tables(0, "1, 2"),
self.__setoftables_2)
self.assertEqual(self.proxy.dump.shard_tables(0, "1,2, 3"),
self.__setoftables_3)
self.assertEqual(self.proxy.dump.shard_maps(0),
self.__setofshardmaps)
self.assertEqual(self.proxy.dump.shard_maps(0, '1'),
self.__setofshardmaps_1)
self.assertEqual(self.proxy.dump.shard_maps(0, '1, 2'),
self.__setofshardmaps_2)
self.assertEqual(self.proxy.dump.shard_maps(0, '1, 2, 3'),
self.__setofshardmaps_3)
self.assertEqual(self.proxy.dump.shard_index(0),
self.__setofshardindexes)
self.assertEqual(self.proxy.dump.shard_index(0, "1"),
self.__setofshardindexes_1)
self.assertEqual(self.proxy.dump.shard_index(0, "1, 2, 3"),
self.__setofshardindexes_3)
self.assertEqual(
self.proxy.dump.shard_index(0, "1, 2, 3, 4, 5"),
self.__setofshardindexes_5
)
self.assertEqual(
self.proxy.dump.sharding_information(0, "db1.t1"),
self.__shardinginformation_1
)
self.assertEqual(
self.proxy.dump.sharding_information(0, "db1.t1, db2.t2"),
self.__shardinginformation_2
)
self.assertEqual(
self.proxy.dump.sharding_information(0, "db1.t1, db2.t2,db3.t3"),
self.__shardinginformation_3
)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestShardingPrune(unittest.TestCase):
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Creates the following topology for testing,
GROUPID1 - localhost:13001, localhost:13002 - Global Group
GROUPID2 - localhost:13003, localhost:13004 - shard 1
GROUPID3 - localhost:13005, localhost:13006 - shard 2
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 601):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 601):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 601):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 601):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 601):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(
1,
"GROUPID2/1,GROUPID3/101,GROUPID4/201,"
"GROUPID5/301,GROUPID6/401",
"ENABLED"
)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
def test_prune_shard(self):
status = self.proxy.sharding.prune_shard("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
status = self.proxy.sharding.lookup_servers("db1.t1", 1, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MAX(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MIN(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 1)
status = self.proxy.sharding.lookup_servers("db1.t1", 101, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MAX(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt(
"SELECT MIN(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 101)
status = self.proxy.sharding.lookup_servers("db1.t1", 202, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MAX(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 300)
rows = shard_server.exec_stmt(
"SELECT MIN(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 201)
status = self.proxy.sharding.lookup_servers("db1.t1", 303, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MAX(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 400)
rows = shard_server.exec_stmt(
"SELECT MIN(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 301)
status = self.proxy.sharding.lookup_servers("db1.t1", 404, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt(
"SELECT MAX(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 600)
rows = shard_server.exec_stmt(
"SELECT MIN(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 401)
def tearDown(self):
self.__server_2.exec_stmt("DROP TABLE db1.t1")
self.__server_2.exec_stmt("DROP DATABASE db1")
self.__server_3.exec_stmt("DROP TABLE db1.t1")
self.__server_3.exec_stmt("DROP DATABASE db1")
self.__server_4.exec_stmt("DROP TABLE db1.t1")
self.__server_4.exec_stmt("DROP DATABASE db1")
self.__server_5.exec_stmt("DROP TABLE db1.t1")
self.__server_5.exec_stmt("DROP DATABASE db1")
self.__server_6.exec_stmt("DROP TABLE db1.t1")
self.__server_6.exec_stmt("DROP DATABASE db1")
status = self.proxy.sharding.disable_shard("1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.disable_shard("2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.disable_shard("3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.disable_shard("4")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.disable_shard("5")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.remove_shard("1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.remove_shard("2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.remove_shard("3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.remove_shard("4")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.remove_shard("5")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.remove_table("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard_mapping).")
status = self.proxy.sharding.remove_definition("1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard_mapping_defn).")
self.proxy.group.demote("GROUPID1")
self.proxy.group.demote("GROUPID2")
self.proxy.group.demote("GROUPID3")
self.proxy.group.demote("GROUPID4")
self.proxy.group.demote("GROUPID5")
self.proxy.group.demote("GROUPID6")
for group_id in ("GROUPID1", "GROUPID2", "GROUPID3",
"GROUPID4", "GROUPID5", "GROUPID6"):
status = self.proxy.group.lookup_servers(group_id)
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
status = \
self.proxy.group.remove(
group_id, obtained_server_list[0]["server_uuid"]
)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_server).")
status = self.proxy.group.destroy(group_id)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_destroy_group).")
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestHashSplitGlobal(unittest.TestCase):
"""Contains unit tests for testing the shard split operation and for
verifying that the global server configuration remains constant after
the shard split configuration.
"""
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Configure the existing environment
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_2.exec_stmt("CREATE DATABASE db3")
self.__server_2.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_3.exec_stmt("CREATE DATABASE db3")
self.__server_3.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_4.exec_stmt("CREATE DATABASE db3")
self.__server_4.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_5.exec_stmt("CREATE DATABASE db3")
self.__server_5.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(1, "db3.t3", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(1, "GROUPID2,GROUPID3,GROUPID4,GROUPID5", "ENABLED")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
status = self.proxy.sharding.prune_shard("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
status = self.proxy.sharding.prune_shard("db2.t2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
status = self.proxy.sharding.prune_shard("db3.t3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
def test_split_shard_1(self):
'''Test the split of shard 1 and the global server configuration
after that. The test splits shard 1 between GROUPID2 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_db3_t3 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("1", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
self.__server_6.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 ==
(row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b)
)
self.assertTrue(
row_cnt_shard_db2_t2 ==
(row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b)
)
self.assertTrue(
row_cnt_shard_db3_t3 ==
(row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b)
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_2(self):
'''Test the split of shard 2 and the global server configuration
after that. The test splits shard 2 between GROUPID3 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_db3_t3 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("2", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
self.__server_6.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 ==
(row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b)
)
self.assertTrue(
row_cnt_shard_db2_t2 ==
(row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b)
)
self.assertTrue(
row_cnt_shard_db3_t3 ==
(row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b)
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_3(self):
'''Test the split of shard 3 and the global server configuration
after that. The test splits shard 3 between GROUPID4 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_db3_t3 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("3", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
self.__server_6.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 ==
(row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b)
)
self.assertTrue(
row_cnt_shard_db2_t2 ==
(row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b)
)
self.assertTrue(
row_cnt_shard_db3_t3 ==
(row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b)
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_4(self):
'''Test the split of shard 4 and the global server configuration
after that. The test splits shard 4 between GROUPID5 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_db3_t3 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("4", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
self.__server_6.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 ==
(row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b)
)
self.assertTrue(
row_cnt_shard_db2_t2 ==
(row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b)
)
self.assertTrue(
row_cnt_shard_db3_t3 ==
(row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b)
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_hash_dump(self):
"""Test the dump of the HASH sharding information for the table.
"""
shardinginformation_1 = [0, 0, 0,
[['db1', 't1', 'userID',
'E2996A7B8509367020B55A4ACD2AE46A',
'1', 'HASH', 'GROUPID2', 'GROUPID1'],
['db1', 't1', 'userID',
'E88F547DF45C99F27646C76316FB21DF',
'2', 'HASH', 'GROUPID3', 'GROUPID1'],
['db1', 't1', 'userID',
'7A2E76448FF04233F3851A492BEF1090',
'3', 'HASH', 'GROUPID4', 'GROUPID1'],
['db1', 't1', 'userID',
'97427AA63E300F56536710F5D73A35FA',
'4', 'HASH', 'GROUPID5', 'GROUPID1']]]
self.assertEqual(
self.proxy.dump.sharding_information(0, "db1.t1"),
shardinginformation_1
)
def tearDown(self):
"""Clean up the existing environment
"""
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.remove_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.remove_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.remove_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.remove_shard(4)
self.proxy.sharding.disable_shard(5)
self.proxy.sharding.remove_shard(5)
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestShardingPrune(tests.utils.TestCase):
def setUp(self):
"""Creates the following topology for testing,
GROUPID1 - localhost:13001, localhost:13002 - Global Group
GROUPID2 - localhost:13003, localhost:13004 - shard 1
GROUPID3 - localhost:13005, localhost:13006 - shard 2
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(3),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add(self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(4),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add(self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(5),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 71):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add(self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(
1, "GROUPID2/1,GROUPID3/101,GROUPID4/1001,"
"GROUPID5/10001,GROUPID6/100001", "ENABLED")
self.check_xmlrpc_command_result(status)
def test_prune_table_not_exists(self):
"""Delete the data in the database and verify that the prune does
not fail once the database has been removed.
"""
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
status = self.proxy.sharding.prune_shard("db1.t1")
def test_prune_shard(self):
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", 1, "LOCAL")
info = self.check_xmlrpc_simple(status, {}, rowcount=1)
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 70)
rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 70)
rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 1)
status = self.proxy.sharding.lookup_servers("db1.t1", 101, "LOCAL")
info = self.check_xmlrpc_simple(status, {}, rowcount=1)
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 300)
rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 101)
status = self.proxy.sharding.lookup_servers("db1.t1", 1202, "LOCAL")
info = self.check_xmlrpc_simple(status, {}, rowcount=1)
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 1200)
rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 1001)
status = self.proxy.sharding.lookup_servers("db1.t1", 11000, "LOCAL")
info = self.check_xmlrpc_simple(status, {}, rowcount=1)
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 10200)
rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 10001)
status = self.proxy.sharding.lookup_servers("db1.t1", 100100, "LOCAL")
info = self.check_xmlrpc_simple(status, {}, rowcount=1)
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 100200)
rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 100001)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestHashSharding(unittest.TestCase):
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEquals( self.__shard_mapping_list, [])
self.__shard_mapping_id_1 = ShardMapping.define("HASH", "GROUPID1")
self.__shard_mapping_1 = ShardMapping.add(
self.__shard_mapping_id_1,
"db1.t1",
"userID"
)
self.__shard_1 = Shards.add("GROUPID2")
self.__shard_2 = Shards.add("GROUPID3")
self.__shard_3 = Shards.add("GROUPID4")
self.__shard_4 = Shards.add("GROUPID5")
self.__shard_5 = Shards.add("GROUPID6")
self.__hash_sharding_specification_1 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
self.__shard_1.shard_id
)
self.__hash_sharding_specification_2 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
self.__shard_2.shard_id
)
self.__hash_sharding_specification_3 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
self.__shard_3.shard_id
)
self.__hash_sharding_specification_4 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
self.__shard_4.shard_id
)
self.__hash_sharding_specification_5 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
self.__shard_5.shard_id
)
def test_hash_lookup(self):
"""Test the hash sharding lookup.
"""
shard_1_cnt = 0
shard_2_cnt = 0
shard_3_cnt = 0
shard_4_cnt = 0
shard_5_cnt = 0
#Lookup a range of keys to ensure that all the shards are
#utilized.
for i in range(0, 1000):
hash_sharding_spec_1 = HashShardingSpecification.lookup(
i,
self.__shard_mapping_id_1,
"HASH"
)
if self.__shard_1.shard_id == hash_sharding_spec_1.shard_id:
shard_1_cnt = shard_1_cnt + 1
elif self.__shard_2.shard_id == hash_sharding_spec_1.shard_id:
shard_2_cnt = shard_2_cnt + 1
elif self.__shard_3.shard_id == hash_sharding_spec_1.shard_id:
shard_3_cnt = shard_3_cnt + 1
elif self.__shard_4.shard_id == hash_sharding_spec_1.shard_id:
shard_4_cnt = shard_4_cnt + 1
elif self.__shard_5.shard_id == hash_sharding_spec_1.shard_id:
shard_5_cnt = shard_5_cnt + 1
#The following will ensure that both the hash shards are utilized
#to store the keys and the values are not skewed in one shard.
self.assertTrue(shard_1_cnt > 0)
self.assertTrue(shard_2_cnt > 0)
self.assertTrue(shard_3_cnt > 0)
self.assertTrue(shard_4_cnt > 0)
self.assertTrue(shard_5_cnt > 0)
def test_hash_remove(self):
"""Test the removal of hash shards.
"""
hash_sharding_specification_1 = HashShardingSpecification.fetch(1)
hash_sharding_specification_2 = HashShardingSpecification.fetch(2)
hash_sharding_specification_3 = HashShardingSpecification.fetch(3)
hash_sharding_specification_4 = HashShardingSpecification.fetch(4)
hash_sharding_specification_5 = HashShardingSpecification.fetch(5)
hash_sharding_specification_1.remove()
hash_sharding_specification_2.remove()
hash_sharding_specification_3.remove()
hash_sharding_specification_4.remove()
hash_sharding_specification_5.remove()
self.__shard_1.remove()
self.__shard_2.remove()
self.__shard_3.remove()
self.__shard_4.remove()
self.__shard_5.remove()
for i in range(0, 10):
hash_sharding_spec = HashShardingSpecification.lookup(
i,
self.__shard_mapping_id_1,
"HASH"
)
self.assertEqual(hash_sharding_spec, None)
def test_list_shard_mapping(self):
"""Test the listing of HASH shards in a shard mapping.
"""
expected_shard_mapping_list1 = [1, "HASH", "GROUPID1"]
obtained_shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEqual(set(expected_shard_mapping_list1),
set(obtained_shard_mapping_list[0]))
def test_shard_mapping_list_mappings(self):
"""Test the listing of all HASH shards in a shard mapping.
"""
shard_mappings = ShardMapping.list("HASH")
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_1, shard_mappings[0]))
def test_fetch_sharding_scheme(self):
"""Test the fetch method of the HASH sharding scheme.
"""
hash_sharding_specification_1 = HashShardingSpecification.fetch(1)
hash_sharding_specification_2 = HashShardingSpecification.fetch(2)
hash_sharding_specification_3 = HashShardingSpecification.fetch(3)
hash_sharding_specification_4 = HashShardingSpecification.fetch(4)
hash_sharding_specification_5 = HashShardingSpecification.fetch(5)
hash_sharding_specifications = HashShardingSpecification.list(1)
#list does not return the hashing specifications in order of shard_id,
#hence a direct comparison is not posssible.
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_1,
hash_sharding_specifications
)
)
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_2,
hash_sharding_specifications
)
)
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_3,
hash_sharding_specifications
)
)
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_4,
hash_sharding_specifications
)
)
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_5,
hash_sharding_specifications
)
)
def test_prune_shard(self):
rows = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 500)
status = self.proxy.sharding.prune_shard("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
rows = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
cnt1 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
cnt2 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
cnt3 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
cnt4 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
cnt5 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
self.assertTrue((cnt1 + cnt2 + cnt3 + cnt4 + cnt5) == 500)
def test_prune_lookup(self):
self.proxy.sharding.prune_shard("db1.t1")
rows = self.__server_2.exec_stmt(
"SELECT userID FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(
val,
self.__shard_mapping_id_1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(
val,
self.__shard_mapping_id_1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(
val,
self.__shard_mapping_id_1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(
val,
self.__shard_mapping_id_1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(
val,
self.__shard_mapping_id_1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def hash_sharding_specification_in_list(self,
hash_sharding_spec,
hash_sharding_specification_list
):
"""Verify if the given hash sharding specification is present in
the list of hash sharding specifications.
:param hash_sharding_spec: The hash sharding specification that
needs to be lookedup.
:param hash_sharding_specification_list: The list of hash sharding
specifications
:return: True if the hash sharding specification is present.
False otherwise.
"""
for i in range(0, len(hash_sharding_specification_list)):
if ShardingUtils.compare_hash_specifications(
hash_sharding_spec,
hash_sharding_specification_list[i]
):
return True
return False
def tearDown(self):
"""Tear down the state store setup.
"""
self.__server_2.exec_stmt("DROP TABLE db1.t1")
self.__server_2.exec_stmt("DROP DATABASE db1")
self.__server_3.exec_stmt("DROP TABLE db1.t1")
self.__server_3.exec_stmt("DROP DATABASE db1")
self.__server_4.exec_stmt("DROP TABLE db1.t1")
self.__server_4.exec_stmt("DROP DATABASE db1")
self.__server_5.exec_stmt("DROP TABLE db1.t1")
self.__server_5.exec_stmt("DROP DATABASE db1")
self.__server_6.exec_stmt("DROP TABLE db1.t1")
self.__server_6.exec_stmt("DROP DATABASE db1")
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestHashSplitGlobal(tests.utils.TestCase):
"""Contains unit tests for testing the shard split operation and for
verifying that the global server configuration remains constant after
the shard split configuration.
"""
def setUp(self):
"""Configure the existing environment
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_2.exec_stmt("CREATE DATABASE db3")
self.__server_2.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_3.exec_stmt("CREATE DATABASE db3")
self.__server_3.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(3),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_4.exec_stmt("CREATE DATABASE db3")
self.__server_4.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add(self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(4),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_5.exec_stmt("CREATE DATABASE db3")
self.__server_5.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add(self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(5),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add(self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db3.t3", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(
1, "GROUPID2,GROUPID3,GROUPID4,GROUPID5", "ENABLED")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db2.t2")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db3.t3")
self.check_xmlrpc_command_result(status)
def test_split_shard_1(self):
'''Test the split of shard 1 and the global server configuration
after that. The test splits shard 1 between GROUPID2 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_db3_t3 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("1", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
self.__server_2.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b))
self.assertTrue(
row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b))
self.assertTrue(
row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b))
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_2(self):
'''Test the split of shard 2 and the global server configuration
after that. The test splits shard 2 between GROUPID3 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_db3_t3 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("2", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
self.__server_3.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b))
self.assertTrue(
row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b))
self.assertTrue(
row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b))
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_3(self):
'''Test the split of shard 3 and the global server configuration
after that. The test splits shard 3 between GROUPID4 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_db3_t3 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("3", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
self.__server_4.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b))
self.assertTrue(
row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b))
self.assertTrue(
row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b))
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_4(self):
'''Test the split of shard 4 and the global server configuration
after that. The test splits shard 4 between GROUPID5 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_db3_t3 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("4", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
self.__server_5.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b))
self.assertTrue(
row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b))
self.assertTrue(
row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b))
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_hash_dump(self):
"""Test the dump of the HASH sharding information for the table.
"""
shardinginformation_1 = tests.utils.make_info_result(
[[
'db1', 't1', 'userID', 'E2996A7B8509367020B55A4ACD2AE46A', '1',
'HASH', 'GROUPID2', 'GROUPID1'
],
[
'db1', 't1', 'userID', 'E88F547DF45C99F27646C76316FB21DF',
'2', 'HASH', 'GROUPID3', 'GROUPID1'
],
[
'db1', 't1', 'userID', '7A2E76448FF04233F3851A492BEF1090',
'3', 'HASH', 'GROUPID4', 'GROUPID1'
],
[
'db1', 't1', 'userID', '97427AA63E300F56536710F5D73A35FA',
'4', 'HASH', 'GROUPID5', 'GROUPID1'
]])
packet = self.proxy.dump.sharding_information(0, "db1.t1")
self.check_xmlrpc_result(packet, shardinginformation_1)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestShardingPrune(unittest.TestCase):
"""Contains unit tests for testing the shard split operation and for
verifying that the global server configuration remains constant after
the shard split configuration.
"""
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Creates the topology for testing.
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(
1,
"GROUPID2/1,GROUPID3/101,GROUPID4/1001,GROUPID5/10001",
"ENABLED"
)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
status = self.proxy.sharding.prune_shard("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
def test_split_shard_1(self):
'''Test the split of shard 1 and the global server configuration
after that. The test splits shard 1 between GROUPID2 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_before_split_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_split_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_split_db1_t1 = \
int(row_cnt_shard_before_split_db1_t1[0][0])
row_cnt_shard_before_split_db2_t2 = \
int(row_cnt_shard_before_split_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_split_db1_t1, 70)
self.assertEqual(row_cnt_shard_before_split_db2_t2, 70)
status = self.proxy.sharding.split_shard("1", "GROUPID6", "36")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
row_cnt_shard_after_split_db1_t1_a = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_a = \
int(row_cnt_shard_after_split_db1_t1_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 35)
row_cnt_shard_after_split_db2_t2_a = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_a = \
int(row_cnt_shard_after_split_db2_t2_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 35)
row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_b = \
int(row_cnt_shard_after_split_db1_t1_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 35)
row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_b = \
int(row_cnt_shard_after_split_db2_t2_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 35)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_2(self):
'''Test the split of shard 2 and the global server configuration
after that. The test splits shard 2 between GROUPID3 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_before_split_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_split_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_split_db1_t1 = \
int(row_cnt_shard_before_split_db1_t1[0][0])
row_cnt_shard_before_split_db2_t2 = \
int(row_cnt_shard_before_split_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_split_db1_t1, 200)
self.assertEqual(row_cnt_shard_before_split_db2_t2, 200)
status = self.proxy.sharding.split_shard("2", "GROUPID6", "201")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
row_cnt_shard_after_split_db1_t1_a = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_a = \
int(row_cnt_shard_after_split_db1_t1_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 100)
row_cnt_shard_after_split_db2_t2_a = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_a = \
int(row_cnt_shard_after_split_db2_t2_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 100)
row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_b = \
int(row_cnt_shard_after_split_db1_t1_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 100)
row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_b = \
int(row_cnt_shard_after_split_db2_t2_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 100)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_3(self):
'''Test the split of shard 3 and the global server configuration
after that. The test splits shard 3 between GROUPID4 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_before_split_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_split_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_split_db1_t1 = \
int(row_cnt_shard_before_split_db1_t1[0][0])
row_cnt_shard_before_split_db2_t2 = \
int(row_cnt_shard_before_split_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_split_db1_t1, 200)
self.assertEqual(row_cnt_shard_before_split_db2_t2, 200)
status = self.proxy.sharding.split_shard("3", "GROUPID6", "1101")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
row_cnt_shard_after_split_db1_t1_a = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_a = \
int(row_cnt_shard_after_split_db1_t1_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 100)
row_cnt_shard_after_split_db2_t2_a = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_a = \
int(row_cnt_shard_after_split_db2_t2_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 100)
row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_b = \
int(row_cnt_shard_after_split_db1_t1_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 100)
row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_b = \
int(row_cnt_shard_after_split_db2_t2_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 100)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_4(self):
'''Test the split of shard 4 and the global server configuration
after that. The test splits shard 4 between GROUPID5 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_before_split_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_split_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_split_db1_t1 = \
int(row_cnt_shard_before_split_db1_t1[0][0])
row_cnt_shard_before_split_db2_t2 = \
int(row_cnt_shard_before_split_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_split_db1_t1, 400)
self.assertEqual(row_cnt_shard_before_split_db2_t2, 400)
status = self.proxy.sharding.split_shard("4", "GROUPID6", "100001")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
row_cnt_shard_after_split_db1_t1_a = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_a = \
int(row_cnt_shard_after_split_db1_t1_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 200)
row_cnt_shard_after_split_db2_t2_a = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_a = \
int(row_cnt_shard_after_split_db2_t2_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 200)
row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_b = \
int(row_cnt_shard_after_split_db1_t1_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 200)
row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_b = \
int(row_cnt_shard_after_split_db2_t2_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 200)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_error_1(self):
'''Test that splitting with an invalid split value fails to
split a shard.
'''
status = self.proxy.sharding.split_shard("3", "GROUPID6", "10")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_check_shard_information).")
def test_split_shard_error_2(self):
'''Test that splitting with an invalid split value fails to
split a shard.
'''
status = self.proxy.sharding.split_shard("3", "GROUPID6", "1101.5")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_check_shard_information).")
def test_split_shard_error_3(self):
'''Test that splitting with an invalid split value fails to
split a shard.
'''
self.__group_6.status = Group.INACTIVE
status = self.proxy.sharding.split_shard("3", "GROUPID6", "1101.5")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_check_shard_information).")
def test_split_shard_error_4(self):
'''Test that splitting with an empty split value fails to
split a shard.
'''
self.__group_6.status = Group.INACTIVE
status = self.proxy.sharding.split_shard("3", "GROUPID6", None)
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_check_shard_information).")
def tearDown(self):
status = self.proxy.sharding.disable_shard("1")
status = self.proxy.sharding.disable_shard("2")
status = self.proxy.sharding.disable_shard("3")
status = self.proxy.sharding.disable_shard("4")
status = self.proxy.sharding.disable_shard("5")
status = self.proxy.sharding.disable_shard("6")
status = self.proxy.sharding.remove_shard("1")
status = self.proxy.sharding.remove_shard("2")
status = self.proxy.sharding.remove_shard("3")
status = self.proxy.sharding.remove_shard("4")
status = self.proxy.sharding.remove_shard("5")
status = self.proxy.sharding.remove_shard("6")
status = self.proxy.sharding.remove_table("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard_mapping).")
status = self.proxy.sharding.remove_table("db2.t2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard_mapping).")
status = self.proxy.sharding.remove_definition("1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard_mapping_defn).")
self.proxy.group.demote("GROUPID1")
self.proxy.group.demote("GROUPID2")
self.proxy.group.demote("GROUPID3")
self.proxy.group.demote("GROUPID4")
self.proxy.group.demote("GROUPID5")
self.proxy.group.demote("GROUPID6")
for group_id in ("GROUPID1", "GROUPID2", "GROUPID3",
"GROUPID4", "GROUPID5", "GROUPID6"):
status = self.proxy.group.lookup_servers(group_id)
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
status = self.proxy.group.remove(
group_id, obtained_server_list[0]["server_uuid"]
)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_server).")
status = self.proxy.group.destroy(group_id)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_destroy_group).")
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestServerClone(unittest.TestCase):
"""Tests the mysqlfabric clone command to clone all the data in a server
registered in Fabric into another server. Create a Group and add some
servers. Insert data into one of the servers. Now clone this server into
the other servers in the group. Now start replication and ensure everything
works fine.
"""
def assertStatus(self, status, expect):
"""Asserts that the status obtained is similar to the status
expected.
"""
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Create the setup for performing the testing of the server clone
command.
"""
from __main__ import mysqldump_path, mysqlclient_path
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address":MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__options_3 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address":MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
#Insert data into Server 1
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_1.exec_stmt("CREATE DATABASE db1")
self.__server_1.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
self.__server_1.exec_stmt("INSERT INTO db1.t1 "
"VALUES(101, 'TEST 1')")
self.__server_1.exec_stmt("INSERT INTO db1.t1 "
"VALUES(202, 'TEST 2')")
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_1.exec_stmt("CREATE DATABASE db2")
self.__server_1.exec_stmt("CREATE TABLE db2.t1"
"(userID INT, name VARCHAR(30))")
self.__server_1.exec_stmt("INSERT INTO db2.t1 "
"VALUES(101, 'TEST 1')")
self.__server_1.exec_stmt("INSERT INTO db2.t1 "
"VALUES(202, 'TEST 2')")
def test_clone_from_group(self):
"""Verify the clone operation from a group.
"""
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
self.__group_1.add_server(self.__server_2)
self.__group_1.master = self.__server_2.uuid
try:
status = self.proxy.server.clone("GROUPID1", self.__server_2.address,
str(self.__server_1.uuid))
raise Exception("Cloning to a server inside Fabric should "
"throw a fault")
except:
pass
try:
status = self.proxy.server.clone("GROUPID1", self.__server_2.address,
uuid_server3)
raise Exception("Cloning from a server outside the "
"source group should throw a fault")
except:
pass
status = self.proxy.server.clone("GROUPID1", self.__server_3.address,
None)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_restore_server).")
rows = self.__server_3.exec_stmt(
"SELECT NAME FROM db1.t1",
{"fetch" : True})
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
rows = self.__server_3.exec_stmt(
"SELECT NAME FROM db2.t1",
{"fetch" : True})
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
def test_clone_error(self):
"""Verify that the clone operations throws an error
for wrong input values.
"""
try:
status = self.proxy.server.clone(None, self.__server_3.address,
str(self.__server_1.uuid))
raise Exception("Not providing a group "\
"should throw a fault")
except:
pass
def tearDown(self):
"""Clean up the existing environment
"""
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestShardingServices(unittest.TestCase):
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Configure the existing environment
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
self.__group_1.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_3)
self.__group_2.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_3)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_5)
self.__group_3.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_5)
status = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(1, "GROUPID2/0,GROUPID3/1001",
"ENABLED")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
#Ensure that adding an invalid shard_mapping_id fails.
status = self.proxy.sharding.add_shard(20000, "GROUPID2/0", "ENABLED")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_add_shard).")
self.__group_4 = Group("GROUPID4", "FOURTH DUMMY DESCRIPTION")
Group.add(self.__group_4)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
def test_fail_duplicate_add_shard(self):
"""Tests that addition of an existing lower_bound to a
shard mapping fails. Also test adding lower_bounds with
a 0 pre-pended the value being inserted.
"""
status = self.proxy.sharding.add_shard(1, "GROUPID2/0", "ENABLED")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(
status[1][-1]["description"],
"Tried to execute action (_add_shard)."
)
#Since the lower_bound datatype is a VARBINARY, ensure that
#pre-pending a 00 to the lower bound does not result in adding
#the same values in the data store. This basically shows that
#comparisons of integers are not impacted with a 00 pre-pended
#to the values.
status = self.proxy.sharding.add_shard(1, "GROUPID3/001001", "ENABLED")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(
status[1][-1]["description"],
"Tried to execute action (_add_shard)."
)
#Since the lower_bound datatype is a VARBINARY, ensure that
#pre-pending a 0000 to the lower bound does not result in adding
#the same values in the data store. This basically shows that
#comparisons of integers are not impacted with a 00 pre-pended
#to the values.
status = self.proxy.sharding.add_shard(1, "GROUPID3/00001001",
"ENABLED")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(
status[1][-1]["description"],
"Tried to execute action (_add_shard)."
)
def test_define_shard_mapping_wrong_sharding_type(self):
#Use an invalid sharding type (WRONG)
status = self.proxy.sharding.create_definition("WRONG", "GROUPID4")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_define_shard_mapping).")
def test_add_shard_invalid_group_exception(self):
#Use an invalid group ID (WRONG_GROUP)
status = self.proxy.sharding.add_shard(4, "WRONG_GROUP/8001",
"ENABLED")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_add_shard).")
def test_add_shard_invalid_state_exception(self):
#WRONG_STATE is an invalid description of the shard state.
status = self.proxy.sharding.add_shard(4, "GROUP4/8001", "WRONG_STATE")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_add_shard).")
def test_add_shard_invalid_range_exception(self):
#Notice LB > UB in the case below.
status = self.proxy.sharding.add_shard(4, "GROUP4/9000", "ENABLED")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_add_shard).")
def test_add_shard_invalid_shard_mapping(self):
status = self.proxy.sharding.add_shard(25000, "GROUPID4/8001",
"ENABLED")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_add_shard).")
status = self.proxy.sharding.add_shard(25000, "GROUPID4/8001",
"ENABLED")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_add_shard).")
def test_remove_shard_mapping(self):
#Remove the shards before removing the mapping
self.proxy.sharding.disable_shard("1")
self.proxy.sharding.remove_shard("1")
self.proxy.sharding.disable_shard("2")
self.proxy.sharding.remove_shard("2")
status = self.proxy.sharding.remove_table("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard_mapping).")
status = self.proxy.sharding.lookup_table("db1.t1")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], {"shard_mapping_id":"",
"table_name":"",
"column_name":"",
"type_name":"",
"global_group":""})
def test_remove_shard_mapping_shards_exist_exception(self):
status = self.proxy.sharding.remove_table("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard_mapping).")
def test_remove_shard_mapping_exception(self):
status = self.proxy.sharding.remove_table("Wrong")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_remove_shard_mapping).")
def test_remove_sharding_specification(self):
expected_server_list = [
[str(self.__server_5.uuid),
MySQLInstances().get_address(4),
True],
[str(self.__server_6.uuid),
MySQLInstances().get_address(5),
False]
]
status = self.proxy.sharding.disable_shard(1)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.remove_shard(1)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
self.assertEqual(status[0], False)
self.assertNotEqual(status[1], "")
self.assertEqual(status[2], True)
def test_remove_sharding_specification_exception(self):
status = self.proxy.sharding.remove_shard(1)
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_remove_shard).")
def test_remove_sharding_specification_wrong_key_exception(self):
status = self.proxy.sharding.remove_shard(55500)
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_remove_shard).")
def test_lookup_shard_mapping(self):
status = self.proxy.sharding.lookup_table("db1.t1")
expected_output = {"shard_mapping_id":1,
"table_name":"db1.t1",
"column_name":"userID1",
"type_name":"RANGE",
"global_group":"GROUPID1"}
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], expected_output)
def test_lookup_shard_mapping_empty(self):
status = self.proxy.sharding.lookup_table("Wrong")
expected_output = {"shard_mapping_id":"",
"table_name":"",
"column_name":"",
"type_name":"",
"global_group":""}
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], expected_output)
def test_list(self):
status = self.proxy.sharding.list_tables("RANGE")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], [{"shard_mapping_id":1,
"table_name":"db1.t1",
"column_name":"userID1",
"type_name":"RANGE",
"global_group":"GROUPID1"}])
def test_list_exception(self):
status = self.proxy.sharding.list_tables("Wrong")
self.assertEqual(status[0], False)
self.assertNotEqual(status[1], "")
self.assertEqual(status[2], True)
def test_lookup(self):
expected_server_list = [
[str(self.__server_3.uuid),
MySQLInstances().get_address(2),
True],
[str(self.__server_4.uuid),
MySQLInstances().get_address(3),
False]
]
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
expected_uuid_list = [expected_server_list[0][0],
expected_server_list[1][0]]
obtained_uuid_list = [obtained_server_list[0][0],
obtained_server_list[1][0]]
expected_address_list = [expected_server_list[0][1],
expected_server_list[1][1]]
obtained_address_list = [obtained_server_list[0][1],
obtained_server_list[1][1]]
self.assertEqual(set(expected_uuid_list), set(obtained_uuid_list))
self.assertEqual(set(expected_address_list), set(obtained_address_list))
def test_lookup_first_key(self):
expected_server_list = [
[str(self.__server_3.uuid),
MySQLInstances().get_address(2),
True],
[str(self.__server_4.uuid),
MySQLInstances().get_address(3),
False]
]
status = self.proxy.sharding.lookup_servers("db1.t1", 1, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
expected_uuid_list = [expected_server_list[0][0],
expected_server_list[1][0]]
obtained_uuid_list = [obtained_server_list[0][0],
obtained_server_list[1][0]]
expected_address_list = [expected_server_list[0][1],
expected_server_list[1][1]]
obtained_address_list = [obtained_server_list[0][1],
obtained_server_list[1][1]]
self.assertEqual(set(expected_uuid_list), set(obtained_uuid_list))
self.assertEqual(set(expected_address_list), set(obtained_address_list))
def test_lookup_last_key(self):
expected_server_list = [
[str(self.__server_3.uuid),
MySQLInstances().get_address(2),
True],
[str(self.__server_4.uuid),
MySQLInstances().get_address(3),
False]
]
status = self.proxy.sharding.lookup_servers("db1.t1", 1000, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
expected_uuid_list = [expected_server_list[0][0],
expected_server_list[1][0]]
obtained_uuid_list = [obtained_server_list[0][0],
obtained_server_list[1][0]]
expected_address_list = [expected_server_list[0][1],
expected_server_list[1][1]]
obtained_address_list = [obtained_server_list[0][1],
obtained_server_list[1][1]]
self.assertEqual(set(expected_uuid_list), set(obtained_uuid_list))
self.assertEqual(set(expected_address_list), set(obtained_address_list))
def test_lookup_first_in_next_key(self):
expected_server_list = [
[str(self.__server_5.uuid),
MySQLInstances().get_address(4),
True],
[str(self.__server_6.uuid),
MySQLInstances().get_address(5),
False]
]
status = self.proxy.sharding.lookup_servers("db1.t1", 1001, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
expected_uuid_list = [expected_server_list[0][0],
expected_server_list[1][0]]
obtained_uuid_list = [obtained_server_list[0][0],
obtained_server_list[1][0]]
expected_address_list = [expected_server_list[0][1],
expected_server_list[1][1]]
obtained_address_list = [obtained_server_list[0][1],
obtained_server_list[1][1]]
self.assertEqual(set(expected_uuid_list), set(obtained_uuid_list))
self.assertEqual(set(expected_address_list), set(obtained_address_list))
def test_lookup_disabled_exception(self):
self.proxy.sharding.disable_shard(1)
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
self.assertEqual(status[0], False)
self.assertNotEqual(status[1], "")
self.assertEqual(status[2], True)
def test_lookup_wrong_table_exception(self):
status = self.proxy.sharding.lookup_servers("Wrong", 500, "LOCAL")
self.assertEqual(status[0], False)
self.assertNotEqual(status[1], "")
self.assertEqual(status[2], True)
def test_lookup_wrong_key_exception(self):
expected_server_list = [
[str(self.__server_5.uuid),
MySQLInstances().get_address(4),
True],
[str(self.__server_6.uuid),
MySQLInstances().get_address(5),
False]
]
status = self.proxy.sharding.lookup_servers("db1.t1", 55500, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
expected_uuid_list = [expected_server_list[0][0],
expected_server_list[1][0]]
obtained_uuid_list = [obtained_server_list[0][0],
obtained_server_list[1][0]]
expected_address_list = [expected_server_list[0][1],
expected_server_list[1][1]]
obtained_address_list = [obtained_server_list[0][1],
obtained_server_list[1][1]]
self.assertEqual(set(expected_uuid_list), set(obtained_uuid_list))
self.assertEqual(set(expected_address_list), set(obtained_address_list))
def test_list_shard_mappings(self):
expected_shard_mapping_list1 = [1, "RANGE", "GROUPID1"]
status = self.proxy.sharding.list_definitions()
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_shard_mapping_list = status[2]
self.assertEqual(
set(expected_shard_mapping_list1), set(obtained_shard_mapping_list[0])
)
def test_enable_shard_exception(self):
status = self.proxy.sharding.enable_shard(25000)
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_enable_shard).")
def test_disable_shard_exception(self):
status = self.proxy.sharding.disable_shard(25000)
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_disable_shard).")
class TestHashSharding(tests.utils.TestCase):
def setUp(self):
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(3),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add(self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(4),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add(self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(5),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add(self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEquals(self.__shard_mapping_list, [])
self.__shard_mapping_id_1 = ShardMapping.define("HASH", "GROUPID1")
self.__shard_mapping_1 = ShardMapping.add(self.__shard_mapping_id_1,
"db1.t1", "userID")
self.__shard_1 = Shards.add("GROUPID2")
self.__shard_2 = Shards.add("GROUPID3")
self.__shard_3 = Shards.add("GROUPID4")
self.__shard_4 = Shards.add("GROUPID5")
self.__shard_5 = Shards.add("GROUPID6")
self.__hash_sharding_specification_1 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, self.__shard_1.shard_id)
self.__hash_sharding_specification_2 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, self.__shard_2.shard_id)
self.__hash_sharding_specification_3 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, self.__shard_3.shard_id)
self.__hash_sharding_specification_4 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, self.__shard_4.shard_id)
self.__hash_sharding_specification_5 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, self.__shard_5.shard_id)
def test_hash_lookup(self):
"""Test the hash sharding lookup.
"""
shard_1_cnt = 0
shard_2_cnt = 0
shard_3_cnt = 0
shard_4_cnt = 0
shard_5_cnt = 0
#Lookup a range of keys to ensure that all the shards are
#utilized.
for i in range(0, 1000):
hash_sharding_spec_1 = HashShardingSpecification.lookup(
i, self.__shard_mapping_id_1, "HASH")
if self.__shard_1.shard_id == hash_sharding_spec_1.shard_id:
shard_1_cnt = shard_1_cnt + 1
elif self.__shard_2.shard_id == hash_sharding_spec_1.shard_id:
shard_2_cnt = shard_2_cnt + 1
elif self.__shard_3.shard_id == hash_sharding_spec_1.shard_id:
shard_3_cnt = shard_3_cnt + 1
elif self.__shard_4.shard_id == hash_sharding_spec_1.shard_id:
shard_4_cnt = shard_4_cnt + 1
elif self.__shard_5.shard_id == hash_sharding_spec_1.shard_id:
shard_5_cnt = shard_5_cnt + 1
#The following will ensure that both the hash shards are utilized
#to store the keys and the values are not skewed in one shard.
self.assertTrue(shard_1_cnt > 0)
self.assertTrue(shard_2_cnt > 0)
self.assertTrue(shard_3_cnt > 0)
self.assertTrue(shard_4_cnt > 0)
self.assertTrue(shard_5_cnt > 0)
def test_hash_remove(self):
"""Test the removal of hash shards.
"""
hash_sharding_specification_1 = HashShardingSpecification.fetch(1)
hash_sharding_specification_2 = HashShardingSpecification.fetch(2)
hash_sharding_specification_3 = HashShardingSpecification.fetch(3)
hash_sharding_specification_4 = HashShardingSpecification.fetch(4)
hash_sharding_specification_5 = HashShardingSpecification.fetch(5)
hash_sharding_specification_1.remove()
hash_sharding_specification_2.remove()
hash_sharding_specification_3.remove()
hash_sharding_specification_4.remove()
hash_sharding_specification_5.remove()
self.__shard_1.remove()
self.__shard_2.remove()
self.__shard_3.remove()
self.__shard_4.remove()
self.__shard_5.remove()
for i in range(0, 10):
hash_sharding_spec = HashShardingSpecification.lookup(
i, self.__shard_mapping_id_1, "HASH")
self.assertEqual(hash_sharding_spec, None)
def test_list_shard_mapping(self):
"""Test the listing of HASH shards in a shard mapping.
"""
expected_shard_mapping_list1 = [1, "HASH", "GROUPID1"]
obtained_shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEqual(set(expected_shard_mapping_list1),
set(obtained_shard_mapping_list[0]))
def test_shard_mapping_list_mappings(self):
"""Test the listing of all HASH shards in a shard mapping.
"""
shard_mappings = ShardMapping.list("HASH")
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_1,
shard_mappings[0]))
def test_fetch_sharding_scheme(self):
"""Test the fetch method of the HASH sharding scheme.
"""
hash_sharding_specification_1 = HashShardingSpecification.fetch(1)
hash_sharding_specification_2 = HashShardingSpecification.fetch(2)
hash_sharding_specification_3 = HashShardingSpecification.fetch(3)
hash_sharding_specification_4 = HashShardingSpecification.fetch(4)
hash_sharding_specification_5 = HashShardingSpecification.fetch(5)
hash_sharding_specifications = HashShardingSpecification.list(1)
#list does not return the hashing specifications in order of shard_id,
#hence a direct comparison is not posssible.
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_1, hash_sharding_specifications))
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_2, hash_sharding_specifications))
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_3, hash_sharding_specifications))
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_4, hash_sharding_specifications))
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_5, hash_sharding_specifications))
def test_prune_shard(self):
rows = self.__server_2.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_3.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_4.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_5.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_6.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 500)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
rows = self.__server_2.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
cnt1 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_3.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
cnt2 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_4.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
cnt3 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_5.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
cnt4 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_6.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
cnt5 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
self.assertTrue((cnt1 + cnt2 + cnt3 + cnt4 + cnt5) == 500)
def test_prune_lookup(self):
self.proxy.sharding.prune_shard("db1.t1")
rows = self.__server_2.exec_stmt("SELECT userID FROM db1.t1",
{"fetch": True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(
val, self.__shard_mapping_id_1, "HASH")
self.assertEqual(hash_sharding_spec_1.shard_id, 1)
rows = self.__server_3.exec_stmt("SELECT userID FROM db1.t1",
{"fetch": True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(
val, self.__shard_mapping_id_1, "HASH")
self.assertEqual(hash_sharding_spec_2.shard_id, 2)
rows = self.__server_4.exec_stmt("SELECT userID FROM db1.t1",
{"fetch": True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(
val, self.__shard_mapping_id_1, "HASH")
self.assertEqual(hash_sharding_spec_3.shard_id, 3)
rows = self.__server_5.exec_stmt("SELECT userID FROM db1.t1",
{"fetch": True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(
val, self.__shard_mapping_id_1, "HASH")
self.assertEqual(hash_sharding_spec_4.shard_id, 4)
rows = self.__server_6.exec_stmt("SELECT userID FROM db1.t1",
{"fetch": True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(
val, self.__shard_mapping_id_1, "HASH")
self.assertEqual(hash_sharding_spec_5.shard_id, 5)
def hash_sharding_specification_in_list(self, hash_sharding_spec,
hash_sharding_specification_list):
"""Verify if the given hash sharding specification is present in
the list of hash sharding specifications.
:param hash_sharding_spec: The hash sharding specification that
needs to be lookedup.
:param hash_sharding_specification_list: The list of hash sharding
specifications
:return: True if the hash sharding specification is present.
False otherwise.
"""
for i in range(0, len(hash_sharding_specification_list)):
if ShardingUtils.compare_hash_specifications(
hash_sharding_spec, hash_sharding_specification_list[i]):
return True
return False
def tearDown(self):
"""Tear down the state store setup.
"""
tests.utils.cleanup_environment()
class TestShardingServices(tests.utils.TestCase):
def setUp(self):
"""Configure the existing environment
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
self.__group_1.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_3 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__options_4 = {
"uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(3),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_3)
self.__group_2.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_3)
self.__options_5 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(4),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__options_6 = {
"uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(5),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_5)
self.__group_3.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_5)
status = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(1, "GROUPID2/0,GROUPID3/1001",
"ENABLED")
self.check_xmlrpc_command_result(status)
self.__group_4 = Group("GROUPID4", "FOURTH DUMMY DESCRIPTION")
Group.add(self.__group_4)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
def test_fail_duplicate_add_shard_mapping_id(self):
"""Verify that a duplicate shard mapping ID inserted
results in an error.
"""
status = self.proxy.sharding.add_table(1, "db1.t1", "userID1")
self.check_xmlrpc_command_result(status, has_error=True)
def test_fail_non_existent_shard_mapping_id(self):
"""Verify that adding a table to a shard mapping ID that does not
exist does not pass.
"""
status = self.proxy.sharding.add_table(3, "db2.t2", "userID2")
self.check_xmlrpc_command_result(status, has_error=True)
def test_fail_duplicate_add_shard(self):
"""Tests that addition of an existing lower_bound to a
shard mapping fails. Also test adding lower_bounds with
a 0 pre-pended the value being inserted.
"""
status = self.proxy.sharding.add_shard(1, "GROUPID2/0", "ENABLED")
self.check_xmlrpc_command_result(status, has_error=True)
#Since the lower_bound datatype is a VARBINARY, ensure that
#pre-pending a 00 to the lower bound does not result in adding
#the same values in the data store. This basically shows that
#comparisons of integers are not impacted with a 00 pre-pended
#to the values.
status = self.proxy.sharding.add_shard(1, "GROUPID3/001001", "ENABLED")
self.check_xmlrpc_command_result(status, has_error=True)
#Since the lower_bound datatype is a VARBINARY, ensure that
#pre-pending a 0000 to the lower bound does not result in adding
#the same values in the data store. This basically shows that
#comparisons of integers are not impacted with a 00 pre-pended
#to the values.
status = self.proxy.sharding.add_shard(1, "GROUPID3/00001001",
"ENABLED")
self.check_xmlrpc_command_result(status, has_error=True)
def test_define_shard_mapping_wrong_sharding_type(self):
#Use an invalid sharding type (WRONG)
status = self.proxy.sharding.create_definition("WRONG", "GROUPID4")
self.check_xmlrpc_command_result(status, has_error=True)
def test_add_shard_invalid_group_exception(self):
#Use an invalid group ID (WRONG_GROUP)
status = self.proxy.sharding.add_shard(4, "WRONG_GROUP/8001",
"ENABLED")
self.check_xmlrpc_command_result(status, has_error=True)
def test_add_shard_invalid_state_exception(self):
#WRONG_STATE is an invalid description of the shard state.
status = self.proxy.sharding.add_shard(4, "GROUP4/8001", "WRONG_STATE")
self.check_xmlrpc_command_result(status, has_error=True)
def test_add_shard_invalid_range_exception(self):
#Notice LB > UB in the case below.
status = self.proxy.sharding.add_shard(4, "GROUP4/9000", "ENABLED")
self.check_xmlrpc_command_result(status, has_error=True)
def test_add_shard_invalid_shard_mapping(self):
status = self.proxy.sharding.add_shard(25000, "GROUPID4/8001",
"ENABLED")
self.check_xmlrpc_command_result(status, has_error=True)
status = self.proxy.sharding.add_shard(25000, "GROUPID4/8001",
"ENABLED")
self.check_xmlrpc_command_result(status, has_error=True)
def test_remove_shard_mapping(self):
#Remove the shards before removing the mapping
self.proxy.sharding.disable_shard("1")
self.proxy.sharding.remove_shard("1")
self.proxy.sharding.disable_shard("2")
self.proxy.sharding.remove_shard("2")
status = self.proxy.sharding.remove_table("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_table("db1.t1")
info = self.check_xmlrpc_simple(status, {})
self.assertEqual(
info, {
"mapping_id": "",
"table_name": "",
"column_name": "",
"type_name": "",
"global_group": ""
})
def test_remove_shard_mapping_shards_exist_exception(self):
status = self.proxy.sharding.remove_table("db1.t1")
self.check_xmlrpc_command_result(status)
def test_remove_shard_mapping_exception(self):
status = self.proxy.sharding.remove_table("Wrong")
self.check_xmlrpc_command_result(status, has_error=True)
def test_remove_sharding_specification(self):
status = self.proxy.sharding.disable_shard(1)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.remove_shard(1)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
self.check_xmlrpc_command_result(status, has_error=True)
def test_remove_sharding_specification_exception(self):
status = self.proxy.sharding.remove_shard(1)
self.check_xmlrpc_command_result(status, has_error=True)
def test_remove_sharding_specification_wrong_key_exception(self):
status = self.proxy.sharding.remove_shard(55500)
self.check_xmlrpc_command_result(status, has_error=True)
def test_lookup_shard_mapping(self):
status = self.proxy.sharding.lookup_table("db1.t1")
expected_output = {
"mapping_id": 1,
"table_name": "db1.t1",
"column_name": "userID1",
"type_name": "RANGE",
"global_group": "GROUPID1"
}
self.check_xmlrpc_simple(status, expected_output)
def test_lookup_shard_mapping_empty(self):
status = self.proxy.sharding.lookup_table("Wrong")
expected_output = {
"mapping_id": "",
"table_name": "",
"column_name": "",
"type_name": "",
"global_group": ""
}
self.check_xmlrpc_simple(status, expected_output)
def test_list(self):
status = self.proxy.sharding.list_tables("RANGE")
expected_output = {
"mapping_id": 1,
"table_name": "db1.t1",
"column_name": "userID1",
"type_name": "RANGE",
"global_group": "GROUPID1"
}
self.check_xmlrpc_simple(status, expected_output)
def test_list_exception(self):
status = self.proxy.sharding.list_tables("Wrong")
self.check_xmlrpc_command_result(status, has_error=True)
def test_lookup(self):
expected_uuid_list = [
str(self.__server_3.uuid),
str(self.__server_4.uuid)
]
expected_address_list = [
MySQLInstances().get_address(2),
MySQLInstances().get_address(3)
]
# Lookup any key
obtained_uuid_list = []
obtained_address_list = []
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
for row in self.check_xmlrpc_iter(status):
obtained_uuid_list.append(row['server_uuid'])
obtained_address_list.append(row['address'])
self.assertEqual(set(expected_uuid_list), set(obtained_uuid_list))
self.assertEqual(set(expected_address_list),
set(obtained_address_list))
# Lookup first key
obtained_uuid_list = []
obtained_address_list = []
status = self.proxy.sharding.lookup_servers("db1.t1", 1, "LOCAL")
for row in self.check_xmlrpc_iter(status):
obtained_uuid_list.append(row['server_uuid'])
obtained_address_list.append(row['address'])
self.assertEqual(set(expected_uuid_list), set(obtained_uuid_list))
self.assertEqual(set(expected_address_list),
set(obtained_address_list))
# Lookup last key
obtained_uuid_list = []
obtained_address_list = []
status = self.proxy.sharding.lookup_servers("db1.t1", 1000, "LOCAL")
for row in self.check_xmlrpc_iter(status):
obtained_uuid_list.append(row['server_uuid'])
obtained_address_list.append(row['address'])
self.assertEqual(set(expected_uuid_list), set(obtained_uuid_list))
self.assertEqual(set(expected_address_list),
set(obtained_address_list))
# Lookup first in next key.
expected_uuid_list = [
str(self.__server_5.uuid),
str(self.__server_6.uuid)
]
expected_address_list = [
MySQLInstances().get_address(4),
MySQLInstances().get_address(5)
]
obtained_uuid_list = []
obtained_address_list = []
status = self.proxy.sharding.lookup_servers("db1.t1", 1001, "LOCAL")
for row in self.check_xmlrpc_iter(status):
obtained_uuid_list.append(row['server_uuid'])
obtained_address_list.append(row['address'])
self.assertEqual(set(expected_uuid_list), set(obtained_uuid_list))
self.assertEqual(set(expected_address_list),
set(obtained_address_list))
# Lookup wrong table
status = self.proxy.sharding.lookup_servers("Wrong", 500, "LOCAL")
self.check_xmlrpc_command_result(status, has_error=True)
# Lookup wrong key
status = self.proxy.sharding.lookup_servers("db1.t1", 55500, "LOCAL")
for row in self.check_xmlrpc_iter(status):
obtained_uuid_list.append(row['server_uuid'])
obtained_address_list.append(row['address'])
self.assertEqual(set(expected_uuid_list), set(obtained_uuid_list))
self.assertEqual(set(expected_address_list),
set(obtained_address_list))
def test_lookup_disabled_exception(self):
self.proxy.sharding.disable_shard(1)
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
self.check_xmlrpc_command_result(status, has_error=True)
def test_list_shard_mappings(self):
expected_shard_mapping = {
'mapping_id': 1,
'type_name': 'RANGE',
'global_group_id': 'GROUPID1'
}
status = self.proxy.sharding.list_definitions()
info = self.check_xmlrpc_simple(status, {})
self.assertEqual(info, expected_shard_mapping)
def test_enable_shard_exception(self):
status = self.proxy.sharding.enable_shard(25000)
self.check_xmlrpc_command_result(status, has_error=True)
def test_disable_shard_exception(self):
status = self.proxy.sharding.disable_shard(25000)
self.check_xmlrpc_command_result(status, has_error=True)
class TestShardingServices(unittest.TestCase):
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Configure the existing environment
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_2.exec_stmt("CREATE DATABASE db3")
self.__server_2.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_2.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_3.exec_stmt("CREATE DATABASE db3")
self.__server_3.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_3.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_4.exec_stmt("CREATE DATABASE db3")
self.__server_4.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_4.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_5.exec_stmt("CREATE DATABASE db3")
self.__server_5.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_5.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_6.exec_stmt("CREATE DATABASE db2")
self.__server_6.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_6.exec_stmt("CREATE DATABASE db3")
self.__server_6.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_6.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(1, "db2.t2", "userID2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(1, "db3.t3", "userID3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(
1,
"GROUPID2,GROUPID3,GROUPID4,GROUPID5,GROUPID6",
"ENABLED"
)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
status = self.proxy.sharding.prune_shard("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
status = self.proxy.sharding.prune_shard("db2.t2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
status = self.proxy.sharding.prune_shard("db3.t3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
def tearDown(self):
"""Clean up the existing environment
"""
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.remove_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.remove_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.remove_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.remove_shard(4)
self.proxy.sharding.disable_shard(5)
self.proxy.sharding.remove_shard(5)
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
def test_prune_lookup_shard1(self):
'''Verify that after the prune the lookup of any pruned value in the
shard results in looking up the same shard.
'''
self.proxy.sharding.prune_shard("db1.t1")
rows = self.__server_2.exec_stmt("SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def test_prune_lookup_shard2(self):
'''Verify that after the prune the lookup of any pruned value in the
shard results in looking up the same shard.
'''
self.proxy.sharding.prune_shard("db2.t2")
rows = self.__server_2.exec_stmt("SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def test_prune_lookup_shard3(self):
'''Verify that after the prune the lookup of any pruned value in the
shard results in looking up the same shard.
'''
self.proxy.sharding.prune_shard("db3.t3")
rows = self.__server_2.exec_stmt("SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(
val,
1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def test_list_shard_mappings(self):
expected_shard_mapping_list1 = [1, "HASH", "GROUPID1"]
status = self.proxy.sharding.list_definitions()
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_shard_mapping_list = status[2]
self.assertEqual(set(expected_shard_mapping_list1),
set(obtained_shard_mapping_list[0]))
def test_lookup_shard_mapping(self):
status = self.proxy.sharding.lookup_table("db1.t1")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], {"shard_mapping_id":1,
"table_name":"db1.t1",
"column_name":"userID1",
"type_name":"HASH",
"global_group":"GROUPID1"})
status = self.proxy.sharding.lookup_table("db2.t2")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], {"shard_mapping_id":1,
"table_name":"db2.t2",
"column_name":"userID2",
"type_name":"HASH",
"global_group":"GROUPID1"})
class TestShardingServices(tests.utils.TestCase):
def setUp(self):
"""Configure the existing environment
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(3),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add(self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(4),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add(self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(5),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add(self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(
1, "GROUPID2,GROUPID3,"
"GROUPID4,GROUPID5,GROUPID6", "ENABLED")
self.check_xmlrpc_command_result(status)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
def test_add_shard_cannot_add_when_shards_exist_exception(self):
status = self.proxy.sharding.add_shard(1, "NON_EXISTENT_GROUP",
"ENABLED")
self.check_xmlrpc_command_result(status, has_error=True)
def test_add_shard_invalid_group_exception(self):
#Remove all the existing shards in the system. Since add shard
#shard will not be allowed when shards exist in the system
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.remove_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.remove_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.remove_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.remove_shard(4)
self.proxy.sharding.disable_shard(5)
self.proxy.sharding.remove_shard(5)
#Use an invalid group ID (WRONG_GROUP)
status = self.proxy.sharding.add_shard(4, "WRONG_GROUP", "ENABLED")
self.check_xmlrpc_command_result(status, has_error=True)
def test_add_shard_invalid_state_exception(self):
#Remove all the existing shards in the system. Since add shard
#shard will not be allowed when shards exist in the system
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.remove_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.remove_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.remove_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.remove_shard(4)
self.proxy.sharding.disable_shard(5)
self.proxy.sharding.remove_shard(5)
#WRONG_STATE is an invalid description of the shard state.
status = self.proxy.sharding.add_shard(4, "GROUP4", "WRONG_STATE")
self.check_xmlrpc_command_result(status, has_error=True)
def test_add_shard_invalid_shard_mapping(self):
status = self.proxy.sharding.add_shard(25000, "GROUPID4", "ENABLED")
self.check_xmlrpc_command_result(status, has_error=True)
def test_remove_shard_mapping_shards_exist(self):
#It is not an error to remove tables from a shard mapping.
status = self.proxy.sharding.remove_table("db1.t1")
self.check_xmlrpc_command_result(status)
def test_remove_sharding_specification(self):
status = self.proxy.sharding.disable_shard(1)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.remove_shard(1)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.disable_shard(2)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.remove_shard(2)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.disable_shard(3)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.remove_shard(3)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.disable_shard(4)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.remove_shard(4)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.disable_shard(5)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.remove_shard(5)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
self.check_xmlrpc_simple(status, {}, has_error=True)
def test_remove_sharding_specification_exception(self):
status = self.proxy.sharding.remove_shard(1)
self.check_xmlrpc_command_result(status, has_error=True)
def test_lookup_shard_mapping(self):
status = self.proxy.sharding.lookup_table("db1.t1")
self.check_xmlrpc_simple(
status, {
"mapping_id": 1,
"table_name": "db1.t1",
"column_name": "userID1",
"type_name": "HASH",
"global_group": "GROUPID1"
})
def test_list(self):
status = self.proxy.sharding.list_tables("HASH")
self.check_xmlrpc_simple(
status, {
"mapping_id": 1,
"table_name": "db1.t1",
"column_name": "userID1",
"type_name": "HASH",
"global_group": "GROUPID1"
})
def test_list_exception(self):
status = self.proxy.sharding.list_tables("Wrong")
self.check_xmlrpc_simple(status, {}, has_error=True)
def test_lookup(self):
shard_hit_count = {}
for i in range(1, 200):
status = self.proxy.sharding.lookup_servers("db1.t1", i, "LOCAL")
for info in self.check_xmlrpc_iter(status):
uuid = info['server_uuid']
if uuid not in shard_hit_count:
shard_hit_count[uuid] = 0
shard_hit_count[uuid] += 1
self.assertEqual(len(shard_hit_count), 5)
for key, count in shard_hit_count.items():
self.assertTrue(count > 0)
def test_lookup_disabled_exception(self):
#Since inhashing it is difficult to guess which shard the value will
#fall into, disable both the shards.
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.disable_shard(5)
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
self.check_xmlrpc_simple(status, {}, has_error=True)
def test_lookup_wrong_table_exception(self):
status = self.proxy.sharding.lookup_servers("Wrong", 500, "LOCAL")
self.check_xmlrpc_simple(status, {}, has_error=True)
def test_list_shard_mappings(self):
status = self.proxy.sharding.list_definitions()
self.check_xmlrpc_simple(status, {
'mapping_id': 1,
'type_name': "HASH",
'global_group_id': "GROUPID1",
},
rowcount=1)
def test_enable_shard_exception(self):
status = self.proxy.sharding.enable_shard(25000)
self.check_xmlrpc_command_result(status, has_error=True)
def test_disable_shard_exception(self):
status = self.proxy.sharding.disable_shard(25000)
self.check_xmlrpc_command_result(status, has_error=True)
class TestShardingPrune(tests.utils.TestCase):
"""Contains unit tests for testing the shard split operation and for
verifying that the global server configuration remains constant after
the shard split configuration.
"""
def setUp(self):
"""Creates the topology for testing.
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20) PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_6.exec_stmt("CREATE DATABASE db2")
self.__server_6.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("RANGE_STRING", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(
1,
"GROUPID2/a,GROUPID3/b,GROUPID4/c,GROUPID5/d,GROUPID6/e",
"ENABLED"
)
self.check_xmlrpc_command_result(status)
def test_prune_shard(self):
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", "a3", "LOCAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 70)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", "b12", "LOCAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 300)
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 300)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", "c35", "LOCAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 200)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", "d21", "LOCAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 600)
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 600)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", "e31", "LOCAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 800)
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 800)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestShardingServices(tests.utils.TestCase):
def setUp(self):
"""Configure the existing environment
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_2.exec_stmt("CREATE DATABASE db3")
self.__server_2.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_2.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_3.exec_stmt("CREATE DATABASE db3")
self.__server_3.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_3.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_4.exec_stmt("CREATE DATABASE db3")
self.__server_4.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_4.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_5.exec_stmt("CREATE DATABASE db3")
self.__server_5.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_5.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_6.exec_stmt("CREATE DATABASE db2")
self.__server_6.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_6.exec_stmt("CREATE DATABASE db3")
self.__server_6.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_6.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db2.t2", "userID2")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db3.t3", "userID3")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(
1,
"GROUPID2,GROUPID3,GROUPID4,GROUPID5,GROUPID6",
"ENABLED"
)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db2.t2")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db3.t3")
self.check_xmlrpc_command_result(status)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
def test_prune_lookup_shard1(self):
'''Verify that after the prune the lookup of any pruned value in the
shard results in looking up the same shard.
'''
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
rows = self.__server_2.exec_stmt("SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def test_prune_lookup_shard2(self):
'''Verify that after the prune the lookup of any pruned value in the
shard results in looking up the same shard.
'''
status = self.proxy.sharding.prune_shard("db2.t2")
self.check_xmlrpc_command_result(status)
rows = self.__server_2.exec_stmt("SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def test_prune_lookup_shard3(self):
'''Verify that after the prune the lookup of any pruned value in the
shard results in looking up the same shard.
'''
status = self.proxy.sharding.prune_shard("db3.t3")
self.check_xmlrpc_command_result(status)
rows = self.__server_2.exec_stmt("SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(
val,
1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def test_list_shard_mappings(self):
expected_shard_mapping = {
'mapping_id' : 1,
'type_name' : 'HASH',
'global_group_id' : 'GROUPID1'
}
status = self.proxy.sharding.list_definitions()
obtained_shard_mapping = self.check_xmlrpc_simple(status, {})
self.assertEqual(expected_shard_mapping, obtained_shard_mapping)
def test_lookup_shard_mapping(self):
expected_shard_mapping_1 = {
'mapping_id' : 1,
'table_name' : 'db1.t1',
'column_name' : 'userID1',
'type_name' : 'HASH',
'global_group' : 'GROUPID1'
}
status = self.proxy.sharding.lookup_table("db1.t1")
obtained_shard_mapping = self.check_xmlrpc_simple(status, {})
self.assertEqual(expected_shard_mapping_1, obtained_shard_mapping)
expected_shard_mapping_2 = {
'mapping_id' : 1,
'table_name' : 'db2.t2',
'column_name' : 'userID2',
'type_name' : 'HASH',
'global_group' : 'GROUPID1'
}
status = self.proxy.sharding.lookup_table("db2.t2")
obtained_shard_mapping = self.check_xmlrpc_simple(status, {})
self.assertEqual(expected_shard_mapping_2, obtained_shard_mapping)
class TestHashMoveGlobal(tests.utils.TestCase):
"""Contains unit tests for testing the shard move operation and for
verifying that the global server configuration remains constant after
the shard move configuration.
"""
def setUp(self):
"""Configure the existing environment
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(
1,
"GROUPID2,GROUPID3,GROUPID4,GROUPID5",
"ENABLED"
)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
def test_move_shard_1(self):
'''Test the move of shard 1 and the global server configuration
after that. The test moves shard 1 from GROUPID2 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID2 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("1", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_2(self):
'''Test the move of shard 2 and the global server configuration
after that. The test moves shard 2 from GROUPID3 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID3 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("2", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_3(self):
'''Test the move of shard 3 and the global server configuration
after that. The test moves shard 3 from GROUPID4 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID4 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("3", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the third shard's
#original group.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard
#new group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_4(self):
'''Test the move of shard 4 and the global server configuration
after that. The test moves shard 4 from GROUPID5 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID5 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("4", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the fourth shard's
#original group.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard's new
#group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestServerClone(tests.utils.TestCase):
"""Tests the mysqlfabric clone command to clone all the data in a server
registered in Fabric into another server. Create a Group and add some
servers. Insert data into one of the servers. Now clone this server into
the other servers in the group. Now start replication and ensure everything
works fine.
"""
def setUp(self):
"""Create the setup for performing the testing of the server clone
command.
"""
_LOGGER.critical("\n\nStart of test-fixture-SetUp(): %s\n" %
(self._testMethodName))
from __main__ import mysqldump_path, mysqlclient_path
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__options_3 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
#Insert data into Server 1
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_1.exec_stmt("CREATE DATABASE db1")
self.__server_1.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
self.__server_1.exec_stmt("INSERT INTO db1.t1 "
"VALUES(101, 'TEST 1')")
self.__server_1.exec_stmt("INSERT INTO db1.t1 "
"VALUES(202, 'TEST 2')")
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_1.exec_stmt("CREATE DATABASE db2")
self.__server_1.exec_stmt("CREATE TABLE db2.t1"
"(userID INT, name VARCHAR(30))")
self.__server_1.exec_stmt("INSERT INTO db2.t1 "
"VALUES(101, 'TEST 1')")
self.__server_1.exec_stmt("INSERT INTO db2.t1 "
"VALUES(202, 'TEST 2')")
_LOGGER.debug("End of test-fixture-SetUp(): %s\n" %
(self._testMethodName))
def test_clone_from_group(self):
"""Verify the clone operation from a group.
"""
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
self.__group_1.add_server(self.__server_2)
self.__group_1.master = self.__server_2.uuid
try:
status = self.proxy.server.clone("GROUPID1",
self.__server_2.address,
str(self.__server_1.uuid))
raise Exception("Cloning to a server inside Fabric should "
"throw a fault")
except:
pass
try:
status = self.proxy.server.clone("GROUPID1",
self.__server_2.address,
uuid_server3)
raise Exception("Cloning from a server outside the "
"source group should throw a fault")
except:
pass
status = self.proxy.server.clone("GROUPID1", self.__server_3.address,
None)
self.check_xmlrpc_command_result(status)
rows = self.__server_3.exec_stmt("SELECT NAME FROM db1.t1",
{"fetch": True})
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
rows = self.__server_3.exec_stmt("SELECT NAME FROM db2.t1",
{"fetch": True})
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
def test_clone_permissions(self):
"""Verify the permissions, needed for a clone operation.
Change password of the backup user, clone fails on backup.
Change password back,
Revoke SELECT from the backup user, clone fails on backup.
Grant SELECT back,
Change password of the restore user, clone fails on restore.
Change password back,
Revoke CREATE from the restore user, clone fails on restore.
Grant CREATE back.
Clone succeeds.
"""
#
# Skip test in trial-mode. Users are the same as admin user.
# Admin won't be able to grant a privilege back to itself.
#
if ((MySQLInstances().backup_user == MySQLInstances().user)
or (MySQLInstances().restore_user == MySQLInstances().user)):
# The trailing comma prevents a newline.
print "Skipping test_clone_permissions in trial-mode --- ",
return
server1 = self.__server_1
server2 = self.__server_2
server3 = self.__server_3
#
# Construct a group in the backing store.
#
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(server1)
self.__group_1.add_server(server2)
self.__group_1.master = server2.uuid
#
# Change password of the the backup user, try a clone, which
# fails, because the Fabric instance does still use the old
# password, change password back. This works, because Fabric
# does not pool the connections for backup and restore users. So
# they have to be freshly connected for backup and restore
# operations.
#
server1.exec_stmt(
"SET PASSWORD FOR '{user}'@'%' ="
" PASSWORD('foobar')".format(user=MySQLInstances().backup_user))
status = self.proxy.server.clone("GROUPID1", server3.address, None)
self.check_xmlrpc_command_result(status, has_error=True)
server1.exec_stmt("SET PASSWORD FOR '{user}'@'%' ="
" PASSWORD('{passwd}')".format(
user=MySQLInstances().backup_user,
passwd=MySQLInstances().backup_passwd))
#
# Revoke the SELECT privilege from the backup user,
# try a clone, which fails, grant SELECT back.
#
server1.exec_stmt("REVOKE SELECT ON *.* FROM '{user}'@'%'".format(
user=MySQLInstances().backup_user))
status = self.proxy.server.clone("GROUPID1", server3.address, None)
self.check_xmlrpc_command_result(status, has_error=True)
server1.exec_stmt("GRANT SELECT ON *.* TO '{user}'@'%'".format(
user=MySQLInstances().backup_user))
#
# Change password of the the restore user, try a clone, which
# fails, because the Fabric instance does still use the old
# password, change password back. This works, because Fabric
# does not pool the connections for backup and restore users. So
# they have to be freshly connected for backup and restore
# operations.
#
server3.exec_stmt(
"SET PASSWORD FOR '{user}'@'%' ="
" PASSWORD('foobar')".format(user=MySQLInstances().restore_user))
status = self.proxy.server.clone("GROUPID1", server3.address, None)
self.check_xmlrpc_command_result(status, has_error=True)
server3.exec_stmt("SET PASSWORD FOR '{user}'@'%' ="
" PASSWORD('{passwd}')".format(
user=MySQLInstances().restore_user,
passwd=MySQLInstances().restore_passwd))
#
# Revoke the CREATE privilege from the restore user,
# try a clone, which fails, grant CREATE back.
#
server3.exec_stmt("REVOKE CREATE ON *.* FROM '{user}'@'%'".format(
user=MySQLInstances().restore_user))
# We have to clear GTID_EXECUTED before we can restore.
reset_master(server3)
status = self.proxy.server.clone("GROUPID1", server3.address, None)
self.check_xmlrpc_command_result(status, has_error=True)
server3.exec_stmt("GRANT CREATE ON *.* TO '{user}'@'%'".format(
user=MySQLInstances().restore_user))
# We have to clear GTID_EXECUTED before we can try the next restore.
reset_master(server3)
#
# Run a successful clone.
#
status = self.proxy.server.clone("GROUPID1", server3.address, None)
self.check_xmlrpc_command_result(status)
rows = server3.exec_stmt("SELECT NAME FROM db1.t1", {"fetch": True})
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
rows = server3.exec_stmt("SELECT NAME FROM db2.t1", {"fetch": True})
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
def test_clone_error(self):
"""Verify that the clone operations throws an error
for wrong input values.
"""
try:
status = self.proxy.server.clone(None, self.__server_3.address,
str(self.__server_1.uuid))
raise Exception("Not providing a group "\
"should throw a fault")
except:
pass
def tearDown(self):
"""Clean up the existing environment
"""
_LOGGER.debug("Start of test-fixture-tearDown(): %s" %
(self._testMethodName))
tests.utils.cleanup_environment()
_LOGGER.debug("End of test-fixture-tearDown(): %s" %
(self._testMethodName))
class TestSharding(tests.utils.TestCase):
"""Test dump interface associated to sharding.
"""
def setUp(self):
"""Configure the existing environment
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": "server_1.mysql.com:3060",
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa75a12a-98d1-414c-96af-9e9d4b179678}"),
"address": "server_2.mysql.com:3060",
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
self.__group_1.add_server(self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__options_4 = {
"uuid": _uuid.UUID("{dd75a12a-98d1-414c-96af-9e9d4b179678}"),
"address": "server_4.mysql.com:3060",
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_3)
self.__group_2.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_3)
self.__options_5 = {
"uuid": _uuid.UUID("{ee75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__options_6 = {
"uuid": _uuid.UUID("{ff75a12a-98d1-414c-96af-9e9d4b179678}"),
"address": "server_6.mysql.com:3060",
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_5)
self.__group_3.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_5)
self.__options_1_host, self.__options_1_port = \
server_utils.split_host_port(self.__options_1["address"], 13001)
self.__options_2_host, self.__options_2_port = \
server_utils.split_host_port(self.__options_2["address"], 13001)
self.__options_3_host, self.__options_3_port = \
server_utils.split_host_port(self.__options_3["address"], 13001)
self.__options_4_host, self.__options_4_port = \
server_utils.split_host_port(self.__options_4["address"], 13001)
self.__options_5_host, self.__options_5_port = \
server_utils.split_host_port(self.__options_5["address"], 13001)
self.__options_6_host, self.__options_6_port = \
server_utils.split_host_port(self.__options_6["address"], 13001)
group_4 = Group("GROUPID4", "4TH description.")
Group.add(group_4)
group_5 = Group("GROUPID5", "5TH description.")
Group.add(group_5)
group_6 = Group("GROUPID6", "6TH description.")
Group.add(group_6)
group_7 = Group("GROUPID7", "7TH description.")
Group.add(group_7)
group_8 = Group("GROUPID8", "8TH description.")
Group.add(group_8)
group_9 = Group("GROUPID9", "9TH description.")
Group.add(group_9)
group_10 = Group("GROUPID10", "10TH description.")
Group.add(group_10)
group_11 = Group("GROUPID11", "11TH description.")
Group.add(group_11)
group_12 = Group("GROUPID12", "12TH description.")
Group.add(group_12)
group_13 = Group("GROUPID13", "13TH description.")
Group.add(group_13)
group_14 = Group("GROUPID14", "14TH description.")
Group.add(group_14)
self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEquals(self.__shard_mapping_list, [])
self.__shard_mapping_id_1 = ShardMapping.define("RANGE", "GROUPID10")
self.__shard_mapping_id_2 = ShardMapping.define("RANGE", "GROUPID11")
self.__shard_mapping_id_3 = ShardMapping.define("RANGE", "GROUPID12")
#Test with sharding type values in lower case
self.__shard_mapping_id_4 = ShardMapping.define("range", "GROUPID13")
self.__shard_mapping_id_5 = ShardMapping.define("range", "GROUPID14")
self.__shard_mapping_1 = \
ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID1")
self.__shard_mapping_2 = \
ShardMapping.add(self.__shard_mapping_id_2, "db2.t2", "userID2")
self.__shard_mapping_3 = \
ShardMapping.add(self.__shard_mapping_id_3, "db3.t3", "userID3")
self.__shard_mapping_4 = \
ShardMapping.add(self.__shard_mapping_id_4, "db4.t4", "userID4")
self.__shard_mapping_5 = \
ShardMapping.add(self.__shard_mapping_id_5, "prune_db.prune_table",
"userID")
self.__shard_id_1 = Shards.add("GROUPID1", "ENABLED")
self.__shard_id_2 = Shards.add("GROUPID10", "ENABLED")
self.__shard_id_3 = Shards.add("GROUPID11", "DISABLED")
self.__shard_id_4 = Shards.add("GROUPID4", "ENABLED")
self.__shard_id_5 = Shards.add("GROUPID5", "ENABLED")
self.__shard_id_6 = Shards.add("GROUPID6", "ENABLED")
self.__shard_id_7 = Shards.add("GROUPID7", "ENABLED")
self.__shard_id_8 = Shards.add("GROUPID8", "ENABLED")
self.__shard_id_9 = Shards.add("GROUPID9", "ENABLED")
self.__shard_id_10 = Shards.add("GROUPID2", "ENABLED")
self.__shard_id_11 = Shards.add("GROUPID3", "ENABLED")
self.__range_sharding_specification_1 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, 0,
self.__shard_id_1.shard_id)
self.__range_sharding_specification_2 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, 1001,
self.__shard_id_2.shard_id)
self.__range_sharding_specification_3 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, 2001,
self.__shard_id_3.shard_id)
self.__range_sharding_specification_4 = RangeShardingSpecification.add(
self.__shard_mapping_2.shard_mapping_id, 3001,
self.__shard_id_4.shard_id)
self.__range_sharding_specification_5 = RangeShardingSpecification.add(
self.__shard_mapping_2.shard_mapping_id, 4001,
self.__shard_id_5.shard_id)
self.__range_sharding_specification_6 = RangeShardingSpecification.add(
self.__shard_mapping_3.shard_mapping_id, 6001,
self.__shard_id_6.shard_id)
self.__range_sharding_specification_7 = RangeShardingSpecification.add(
self.__shard_mapping_3.shard_mapping_id, 7001,
self.__shard_id_7.shard_id)
self.__range_sharding_specification_8 = RangeShardingSpecification.add(
self.__shard_mapping_4.shard_mapping_id, 8001,
self.__shard_id_8.shard_id)
self.__range_sharding_specification_9 = RangeShardingSpecification.add(
self.__shard_mapping_4.shard_mapping_id, 10001,
self.__shard_id_9.shard_id)
self.__range_sharding_specification_10 = RangeShardingSpecification.add(
self.__shard_mapping_5.shard_mapping_id, 100,
self.__shard_id_10.shard_id)
self.__range_sharding_specification_11 = RangeShardingSpecification.add(
self.__shard_mapping_5.shard_mapping_id, 201,
self.__shard_id_11.shard_id)
READ_ONLY = MySQLServer.get_mode_idx(MySQLServer.READ_ONLY)
READ_WRITE = MySQLServer.get_mode_idx(MySQLServer.READ_WRITE)
SECONDARY = MySQLServer.get_status_idx(MySQLServer.SECONDARY)
PRIMARY = MySQLServer.get_status_idx(MySQLServer.PRIMARY)
self.__setofservers = tests.utils.make_servers_result(
[[
str(self.__server_1.uuid), 'GROUPID1', self.__options_1_host,
self.__options_1_port, READ_ONLY, SECONDARY, 1.0
],
[
str(self.__server_2.uuid), 'GROUPID1', self.__options_2_host,
self.__options_2_port, READ_ONLY, SECONDARY, 1.0
],
[
str(self.__server_3.uuid), 'GROUPID2', self.__options_3_host,
self.__options_3_port, READ_WRITE, PRIMARY, 1.0
],
[
str(self.__server_4.uuid), 'GROUPID2', self.__options_4_host,
self.__options_4_port, READ_ONLY, SECONDARY, 1.0
],
[
str(self.__server_5.uuid), 'GROUPID3', self.__options_5_host,
self.__options_5_port, READ_WRITE, PRIMARY, 1.0
],
[
str(self.__server_6.uuid), 'GROUPID3', self.__options_6_host,
self.__options_6_port, READ_ONLY, SECONDARY, 1.0
]])
self.__setofservers_1 = tests.utils.make_servers_result(
[[
str(self.__server_1.uuid), 'GROUPID1', self.__options_1_host,
self.__options_1_port, READ_ONLY, SECONDARY, 1.0
],
[
str(self.__server_2.uuid), 'GROUPID1', self.__options_2_host,
self.__options_2_port, READ_ONLY, SECONDARY, 1.0
]])
self.__setofservers_2 = tests.utils.make_servers_result(
[[
str(self.__server_1.uuid), 'GROUPID1', self.__options_1_host,
self.__options_1_port, READ_ONLY, SECONDARY, 1.0
],
[
str(self.__server_2.uuid), 'GROUPID1', self.__options_2_host,
self.__options_2_port, READ_ONLY, SECONDARY, 1.0
],
[
str(self.__server_3.uuid), 'GROUPID2', self.__options_3_host,
self.__options_3_port, READ_WRITE, PRIMARY, 1.0
],
[
str(self.__server_4.uuid), 'GROUPID2', self.__options_4_host,
self.__options_4_port, READ_ONLY, SECONDARY, 1.0
]])
self.__setofservers_3 = tests.utils.make_servers_result(
[[
str(self.__server_1.uuid), 'GROUPID1', self.__options_1_host,
self.__options_1_port, READ_ONLY, SECONDARY, 1.0
],
[
str(self.__server_2.uuid), 'GROUPID1', self.__options_2_host,
self.__options_2_port, READ_ONLY, SECONDARY, 1.0
],
[
str(self.__server_3.uuid), 'GROUPID2', self.__options_3_host,
self.__options_3_port, READ_WRITE, PRIMARY, 1.0
],
[
str(self.__server_4.uuid), 'GROUPID2', self.__options_4_host,
self.__options_4_port, READ_ONLY, SECONDARY, 1.0
],
[
str(self.__server_5.uuid), 'GROUPID3', self.__options_5_host,
self.__options_5_port, READ_WRITE, PRIMARY, 1.0
],
[
str(self.__server_6.uuid), 'GROUPID3', self.__options_6_host,
self.__options_6_port, READ_ONLY, SECONDARY, 1.0
]])
self.__setoftables = tests.utils.make_tables_result(
[['db1', 't1', 'userID1', '1'], ['db2', 't2', 'userID2', '2'],
['db3', 't3', 'userID3', '3'], ['db4', 't4', 'userID4', '4'],
['prune_db', 'prune_table', 'userID', '5']])
self.__setoftables_1 = tests.utils.make_tables_result(
[['db1', 't1', 'userID1', '1']])
self.__setoftables_2 = tests.utils.make_tables_result(
[['db1', 't1', 'userID1', '1'], ['db2', 't2', 'userID2', '2']])
self.__setoftables_3 = tests.utils.make_tables_result(
[['db1', 't1', 'userID1', '1'], ['db2', 't2', 'userID2', '2'],
['db3', 't3', 'userID3', '3']])
self.__setofshardmaps = tests.utils.make_mapping_result(
[['1', 'RANGE', 'GROUPID10'], ['2', 'RANGE', 'GROUPID11'],
['3', 'RANGE', 'GROUPID12'], ['4', 'RANGE', 'GROUPID13'],
['5', 'RANGE', 'GROUPID14']])
self.__setofshardmaps_1 = tests.utils.make_mapping_result(
[['1', 'RANGE', 'GROUPID10']])
self.__setofshardmaps_2 = tests.utils.make_mapping_result(
[['1', 'RANGE', 'GROUPID10'], ['2', 'RANGE', 'GROUPID11']])
self.__setofshardmaps_3 = tests.utils.make_mapping_result(
[['1', 'RANGE', 'GROUPID10'], ['2', 'RANGE', 'GROUPID11'],
['3', 'RANGE', 'GROUPID12']])
self.__setofshardindexes = tests.utils.make_index_result(
[['0', '1', '1', 'GROUPID1'], ['1001', '1', '2', 'GROUPID10'],
['3001', '2', '4', 'GROUPID4'], ['4001', '2', '5', 'GROUPID5'],
['6001', '3', '6', 'GROUPID6'], ['7001', '3', '7', 'GROUPID7'],
['8001', '4', '8', 'GROUPID8'], ['10001', '4', '9', 'GROUPID9'],
['100', '5', '10', 'GROUPID2'], ['201', '5', '11', 'GROUPID3']])
self.__setofshardindexes_1 = tests.utils.make_index_result(
[['0', '1', '1', 'GROUPID1'], ['1001', '1', '2', 'GROUPID10']])
self.__setofshardindexes_3 = tests.utils.make_index_result(
[['0', '1', '1', 'GROUPID1'], ['1001', '1', '2', 'GROUPID10'],
['3001', '2', '4', 'GROUPID4'], ['4001', '2', '5', 'GROUPID5'],
['6001', '3', '6', 'GROUPID6'], ['7001', '3', '7', 'GROUPID7']])
self.__setofshardindexes_5 = tests.utils.make_index_result(
[['0', '1', '1', 'GROUPID1'], ['1001', '1', '2', 'GROUPID10'],
['3001', '2', '4', 'GROUPID4'], ['4001', '2', '5', 'GROUPID5'],
['6001', '3', '6', 'GROUPID6'], ['7001', '3', '7', 'GROUPID7'],
['8001', '4', '8', 'GROUPID8'], ['10001', '4', '9', 'GROUPID9'],
['100', '5', '10', 'GROUPID2'], ['201', '5', '11', 'GROUPID3']])
self.__shardinginformation_1 = tests.utils.make_info_result(
[[
'db1', 't1', 'userID1', '0', '1', 'RANGE', 'GROUPID1',
'GROUPID10'
],
[
'db1', 't1', 'userID1', '1001', '2', 'RANGE', 'GROUPID10',
'GROUPID10'
]])
self.__shardinginformation_2 = tests.utils.make_info_result(
[[
'db1', 't1', 'userID1', '0', '1', 'RANGE', 'GROUPID1',
'GROUPID10'
],
[
'db1', 't1', 'userID1', '1001', '2', 'RANGE', 'GROUPID10',
'GROUPID10'
],
[
'db2', 't2', 'userID2', '3001', '4', 'RANGE', 'GROUPID4',
'GROUPID11'
],
[
'db2', 't2', 'userID2', '4001', '5', 'RANGE', 'GROUPID5',
'GROUPID11'
]])
self.__shardinginformation_3 = tests.utils.make_info_result(
[[
'db1', 't1', 'userID1', '0', '1', 'RANGE', 'GROUPID1',
'GROUPID10'
],
[
'db1', 't1', 'userID1', '1001', '2', 'RANGE', 'GROUPID10',
'GROUPID10'
],
[
'db2', 't2', 'userID2', '3001', '4', 'RANGE', 'GROUPID4',
'GROUPID11'
],
[
'db2', 't2', 'userID2', '4001', '5', 'RANGE', 'GROUPID5',
'GROUPID11'
],
[
'db3', 't3', 'userID3', '6001', '6', 'RANGE', 'GROUPID6',
'GROUPID12'
],
[
'db3', 't3', 'userID3', '7001', '7', 'RANGE', 'GROUPID7',
'GROUPID12'
]])
def test_dumps(self):
"""Test dump interface associated to sharding.
"""
packet = self.proxy.dump.servers(0)
self.check_xmlrpc_result(packet, self.__setofservers)
packet = self.proxy.dump.servers(0, "GROUPID1")
self.check_xmlrpc_result(packet, self.__setofservers_1)
packet = self.proxy.dump.servers(0, "GROUPID1,GROUPID2")
self.check_xmlrpc_result(packet, self.__setofservers_2)
packet = self.proxy.dump.servers(0, "GROUPID1,GROUPID2,GROUPID3")
self.check_xmlrpc_result(packet, self.__setofservers_3)
packet = self.proxy.dump.shard_tables(0)
self.check_xmlrpc_result(packet, self.__setoftables)
packet = self.proxy.dump.shard_tables(0, "1")
self.check_xmlrpc_result(packet, self.__setoftables_1)
packet = self.proxy.dump.shard_tables(0, "1, 2")
self.check_xmlrpc_result(packet, self.__setoftables_2)
packet = self.proxy.dump.shard_tables(0, "1,2, 3")
self.check_xmlrpc_result(packet, self.__setoftables_3)
packet = self.proxy.dump.shard_maps(0)
self.check_xmlrpc_result(packet, self.__setofshardmaps)
packet = self.proxy.dump.shard_maps(0, '1')
self.check_xmlrpc_result(packet, self.__setofshardmaps_1)
packet = self.proxy.dump.shard_maps(0, '1, 2')
self.check_xmlrpc_result(packet, self.__setofshardmaps_2)
packet = self.proxy.dump.shard_maps(0, '1, 2, 3')
self.check_xmlrpc_result(packet, self.__setofshardmaps_3)
packet = self.proxy.dump.shard_index(0)
self.check_xmlrpc_result(packet, self.__setofshardindexes)
packet = self.proxy.dump.shard_index(0, "1")
self.check_xmlrpc_result(packet, self.__setofshardindexes_1)
packet = self.proxy.dump.shard_index(0, "1, 2, 3")
self.check_xmlrpc_result(packet, self.__setofshardindexes_3)
packet = self.proxy.dump.shard_index(0, "1, 2, 3, 4, 5")
self.check_xmlrpc_result(packet, self.__setofshardindexes_5)
packet = self.proxy.dump.sharding_information(0, "db1.t1")
self.check_xmlrpc_result(packet, self.__shardinginformation_1)
packet = self.proxy.dump.sharding_information(0, "db1.t1, db2.t2")
self.check_xmlrpc_result(packet, self.__shardinginformation_2)
packet = self.proxy.dump.sharding_information(
0, "db1.t1, db2.t2,db3.t3")
self.check_xmlrpc_result(packet, self.__shardinginformation_3)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestSharding(unittest.TestCase):
def setUp(self):
"""Configure the existing environment
"""
self.__options_1 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": "server_1.mysql.com:3060",
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa75a12a-98d1-414c-96af-9e9d4b179678}"),
"address": "server_2.mysql.com:3060",
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
self.__group_1.add_server(self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS prune_db")
self.__server_3.exec_stmt("CREATE DATABASE prune_db")
self.__server_3.exec_stmt("CREATE TABLE prune_db.prune_table"
"(userID INT, name VARCHAR(30))")
self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(101, 'TEST 1')")
self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(202, 'TEST 2')")
self.__options_4 = {
"uuid": _uuid.UUID("{dd75a12a-98d1-414c-96af-9e9d4b179678}"),
"address": "server_4.mysql.com:3060",
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_3)
self.__group_2.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_3)
self.__options_5 = {
"uuid": _uuid.UUID("{ee75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS prune_db")
self.__server_5.exec_stmt("CREATE DATABASE prune_db")
self.__server_5.exec_stmt("CREATE TABLE prune_db.prune_table"
"(userID INT, name VARCHAR(30))")
self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(101, 'TEST 1')")
self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(202, 'TEST 2')")
self.__options_6 = {
"uuid": _uuid.UUID("{ff75a12a-98d1-414c-96af-9e9d4b179678}"),
"address": "server_6.mysql.com:3060",
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_5)
self.__group_3.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_5)
group_4 = Group("GROUPID4", "4TH description.")
Group.add(group_4)
group_5 = Group("GROUPID5", "5TH description.")
Group.add(group_5)
group_6 = Group("GROUPID6", "6TH description.")
Group.add(group_6)
group_7 = Group("GROUPID7", "7TH description.")
Group.add(group_7)
group_8 = Group("GROUPID8", "8TH description.")
Group.add(group_8)
group_9 = Group("GROUPID9", "9TH description.")
Group.add(group_9)
group_10 = Group("GROUPID10", "10TH description.")
Group.add(group_10)
group_11 = Group("GROUPID11", "11TH description.")
Group.add(group_11)
group_12 = Group("GROUPID12", "12TH description.")
Group.add(group_12)
group_13 = Group("GROUPID13", "13TH description.")
Group.add(group_13)
group_14 = Group("GROUPID14", "14TH description.")
Group.add(group_14)
self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEquals(self.__shard_mapping_list, [])
self.__shard_mapping_id_1 = ShardMapping.define("RANGE", "GROUPID10")
self.__shard_mapping_id_2 = ShardMapping.define("RANGE", "GROUPID11")
self.__shard_mapping_id_3 = ShardMapping.define("RANGE", "GROUPID12")
#Test with sharding type values in lower case
self.__shard_mapping_id_4 = ShardMapping.define("range", "GROUPID13")
self.__shard_mapping_id_5 = ShardMapping.define("range", "GROUPID14")
self.__shard_mapping_1 = \
ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID1")
self.__shard_mapping_2 = \
ShardMapping.add(self.__shard_mapping_id_2, "db2.t2", "userID2")
self.__shard_mapping_3 = \
ShardMapping.add(self.__shard_mapping_id_3, "db3.t3", "userID3")
self.__shard_mapping_4 = \
ShardMapping.add(self.__shard_mapping_id_4, "db4.t4", "userID4")
self.__shard_mapping_5 = \
ShardMapping.add(self.__shard_mapping_id_5, "prune_db.prune_table",
"userID")
self.__shard_id_1 = Shards.add("GROUPID1")
self.__shard_id_2 = Shards.add("GROUPID10")
self.__shard_id_3 = Shards.add("GROUPID11")
self.__shard_id_4 = Shards.add("GROUPID4")
self.__shard_id_5 = Shards.add("GROUPID5")
self.__shard_id_6 = Shards.add("GROUPID6")
self.__shard_id_7 = Shards.add("GROUPID7")
self.__shard_id_8 = Shards.add("GROUPID8")
self.__shard_id_9 = Shards.add("GROUPID9")
self.__shard_id_10 = Shards.add("GROUPID2")
self.__shard_id_11 = Shards.add("GROUPID3")
self.__range_sharding_specification_1 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, 0,
self.__shard_id_1.shard_id)
self.__range_sharding_specification_2 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, 1001,
self.__shard_id_2.shard_id)
self.__range_sharding_specification_3 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, 2001,
self.__shard_id_3.shard_id)
self.__range_sharding_specification_4 = RangeShardingSpecification.add(
self.__shard_mapping_2.shard_mapping_id, 3001,
self.__shard_id_4.shard_id)
self.__range_sharding_specification_5 = RangeShardingSpecification.add(
self.__shard_mapping_2.shard_mapping_id, 4001,
self.__shard_id_5.shard_id)
self.__range_sharding_specification_6 = RangeShardingSpecification.add(
self.__shard_mapping_3.shard_mapping_id, 6001,
self.__shard_id_6.shard_id)
self.__range_sharding_specification_7 = RangeShardingSpecification.add(
self.__shard_mapping_3.shard_mapping_id, 7001,
self.__shard_id_7.shard_id)
self.__range_sharding_specification_8 = RangeShardingSpecification.add(
self.__shard_mapping_4.shard_mapping_id, 8001,
self.__shard_id_8.shard_id)
self.__range_sharding_specification_9 = RangeShardingSpecification.add(
self.__shard_mapping_4.shard_mapping_id, 10001,
self.__shard_id_9.shard_id)
self.__range_sharding_specification_10 = RangeShardingSpecification.add(
self.__shard_mapping_5.shard_mapping_id, 100,
self.__shard_id_10.shard_id)
self.__range_sharding_specification_11 = RangeShardingSpecification.add(
self.__shard_mapping_5.shard_mapping_id, 201,
self.__shard_id_11.shard_id)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
def test_fetch_shard_mapping(self):
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_2 = ShardMapping.fetch("db2.t2")
shard_mapping_3 = ShardMapping.fetch("db3.t3")
shard_mapping_4 = ShardMapping.fetch("db4.t4")
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_1,
shard_mapping_1))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_2,
shard_mapping_2))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_3,
shard_mapping_3))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_4,
shard_mapping_4))
def test_fetch_sharding_scheme(self):
range_sharding_specifications = RangeShardingSpecification.list(1)
self.assertTrue(
ShardingUtils.compare_range_specifications(
range_sharding_specifications[0],
self.__range_sharding_specification_1))
self.assertTrue(
ShardingUtils.compare_range_specifications(
range_sharding_specifications[1],
self.__range_sharding_specification_2))
self.assertTrue(
ShardingUtils.compare_range_specifications(
range_sharding_specifications[2],
self.__range_sharding_specification_3))
def test_lookup_sharding_scheme(self):
r_spec_1 = \
RangeShardingSpecification.lookup(500, self.__shard_mapping_id_1,
"RANGE")
self.assertEqual(r_spec_1.shard_id, self.__shard_id_1.shard_id)
r_spec_2 = \
RangeShardingSpecification.lookup(3500, self.__shard_mapping_id_2,
"RANGE")
self.assertEqual(r_spec_2.shard_id, self.__shard_id_4.shard_id)
r_spec_3 = \
RangeShardingSpecification.lookup(6500, self.__shard_mapping_id_3,
"RANGE")
self.assertEqual(r_spec_3.shard_id, self.__shard_id_6.shard_id)
def test_shard_mapping_list_mappings(self):
shard_mappings = ShardMapping.list("RANGE")
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_1,
shard_mappings[0]))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_2,
shard_mappings[1]))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_3,
shard_mappings[2]))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_4,
shard_mappings[3]))
def test_shard_mapping_getters(self):
self.assertEqual(self.__shard_mapping_1.table_name, "db1.t1")
self.assertEqual(self.__shard_mapping_1.column_name, "userID1")
self.assertEqual(self.__shard_mapping_1.type_name, "RANGE")
self.assertEqual(self.__shard_mapping_1.shard_mapping_id, 1)
self.assertEqual(self.__shard_mapping_1.global_group, "GROUPID10")
def test_shard_mapping_remove(self):
self.__range_sharding_specification_1.remove()
self.__range_sharding_specification_2.remove()
self.__range_sharding_specification_3.remove()
self.__range_sharding_specification_4.remove()
self.__range_sharding_specification_5.remove()
self.__range_sharding_specification_6.remove()
self.__range_sharding_specification_7.remove()
self.__range_sharding_specification_8.remove()
self.__range_sharding_specification_9.remove()
self.__range_sharding_specification_10.remove()
self.__shard_id_1.remove()
self.__shard_id_2.remove()
self.__shard_id_3.remove()
self.__shard_id_4.remove()
self.__shard_id_5.remove()
self.__shard_id_6.remove()
self.__shard_id_7.remove()
self.__shard_id_8.remove()
self.__shard_id_9.remove()
self.__shard_id_10.remove()
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_1.remove()
def test_range_sharding_specification_getters(self):
self.assertEqual(
self.__range_sharding_specification_1.shard_mapping_id, 1)
self.assertEqual(self.__range_sharding_specification_1.lower_bound, 0)
self.assertEqual(self.__range_sharding_specification_1.shard_id, 1)
def test_list_shard_mapping(self):
expected_shard_mapping_list1 = [1, "RANGE", "GROUPID10"]
expected_shard_mapping_list2 = [2, "RANGE", "GROUPID11"]
expected_shard_mapping_list3 = [3, "RANGE", "GROUPID12"]
expected_shard_mapping_list4 = [4, "RANGE", "GROUPID13"]
expected_shard_mapping_list5 = [5, "RANGE", "GROUPID14"]
obtained_shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEqual(set(expected_shard_mapping_list1),
set(obtained_shard_mapping_list[0]))
self.assertEqual(set(expected_shard_mapping_list2),
set(obtained_shard_mapping_list[1]))
self.assertEqual(set(expected_shard_mapping_list3),
set(obtained_shard_mapping_list[2]))
self.assertEqual(set(expected_shard_mapping_list4),
set(obtained_shard_mapping_list[3]))
self.assertEqual(set(expected_shard_mapping_list5),
set(obtained_shard_mapping_list[4]))
class TestShardingPrune(tests.utils.TestCase):
"""Contains unit tests for testing the shard split operation and for
verifying that the global server configuration remains constant after
the shard split configuration.
"""
def setUp(self):
"""Creates the topology for testing.
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20) PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("RANGE_STRING", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(
1,
"GROUPID2/a,GROUPID3/b,GROUPID4/c,GROUPID5/d",
"ENABLED"
)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
def test_split_shard_4(self):
'''Test the split of shard 4 and the global server configuration
after that. The test splits shard 4 between GROUPID5 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_before_split_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_split_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_split_db1_t1 = \
int(row_cnt_shard_before_split_db1_t1[0][0])
row_cnt_shard_before_split_db2_t2 = \
int(row_cnt_shard_before_split_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_split_db1_t1, 400)
self.assertEqual(row_cnt_shard_before_split_db2_t2, 400)
status = self.proxy.sharding.split_shard("4", "GROUPID6", "e")
self.check_xmlrpc_command_result(status)
self.__server_5.connect()
row_cnt_shard_after_split_db1_t1_a = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_a = \
int(row_cnt_shard_after_split_db1_t1_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 200)
row_cnt_shard_after_split_db2_t2_a = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_a = \
int(row_cnt_shard_after_split_db2_t2_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 200)
row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_b = \
int(row_cnt_shard_after_split_db1_t1_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 200)
row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_b = \
int(row_cnt_shard_after_split_db2_t2_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 200)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID VARCHAR(20), name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES('%s', 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestShardingPrune(tests.utils.TestCase):
"""Contains unit tests for testing the shard move operation and for
verifying that the global server configuration remains constant after
the shard move configuration.
"""
def check_xmlrpc_command_success(self, packet, expect, has_error):
result = _xmlrpc._decode(packet)
self.assertEqual(bool(result.error), has_error,
"Had error '%s'" % result.error)
# Check that it returned the expected result (if one is
# expected), which is in the first result set (containing
# execution result), first row (there is just one for
# procedures), and last column (containing the result).
if expect is not None:
self.assertEqual(result.results[0][0][3], expect)
# If the call was successful, check that there is at least 2
# result sets and that the second result set contain more than
# zero jobs.
if not has_error:
self.assertTrue(len(result.results) > 1, str(result))
self.assertNotEqual(result.results[1].rowcount, 0,
"had %d result sets" % len(result.results))
def setUp(self):
"""Creates the topology for testing.
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__server_1.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF")
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF")
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71, 10):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301, 10):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201, 10):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201, 10):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201, 10):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71, 10):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301, 10):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201, 10):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201, 10):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201, 10):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF")
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71, 10):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301, 10):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201, 10):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201, 10):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201, 10):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71, 10):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301, 10):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201, 10):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201, 10):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201, 10):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(3),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF")
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71, 10):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301, 10):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201, 10):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201, 10):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201, 10):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71, 10):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301, 10):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201, 10):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201, 10):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201, 10):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add(self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(4),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF")
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71, 10):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301, 10):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201, 10):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201, 10):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201, 10):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71, 10):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301, 10):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201, 10):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201, 10):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201, 10):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add(self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(5),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF")
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add(self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
packet = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.check_xmlrpc_command_success(packet, 1, False)
packet = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.check_xmlrpc_command_success(packet, None, False)
packet = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.check_xmlrpc_command_success(packet, None, False)
packet = self.proxy.sharding.add_shard(
1, "GROUPID2/1,GROUPID3/101,GROUPID4/1001,GROUPID5/10001",
"ENABLED")
self.check_xmlrpc_command_success(packet, None, False)
packet = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_success(packet, None, False)
def test_move_shard_1(self):
'''Test the move of shard 1 and the global server configuration
after that. The test moves shard 1 from GROUPID2 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID2 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_before_move_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_move_db1_t1, 7)
self.assertEqual(row_cnt_shard_before_move_db2_t2, 7)
packet = self.proxy.sharding.move_shard("1", "GROUPID6")
self.check_xmlrpc_command_success(packet, None, False)
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_after_move_db1_t1,
row_cnt_shard_before_move_db1_t1)
self.assertEqual(row_cnt_shard_after_move_db2_t2,
row_cnt_shard_before_move_db2_t2)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the first shard's old
#group.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertEqual(global_table_count, 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the first shard's new
#group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_2(self):
'''Test the move of shard 2 and the global server configuration
after that. The test moves shard 2 from GROUPID3 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID3 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_before_move_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_move_db1_t1, 20)
self.assertEqual(row_cnt_shard_before_move_db2_t2, 20)
packet = self.proxy.sharding.move_shard("2", "GROUPID6")
self.check_xmlrpc_command_success(packet, None, False)
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_after_move_db1_t1,
row_cnt_shard_before_move_db1_t1)
self.assertEqual(row_cnt_shard_after_move_db2_t2,
row_cnt_shard_before_move_db2_t2)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the second shard's
#old group.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertEqual(global_table_count, 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard's new
#group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_3(self):
'''Test the move of shard 3 and the global server configuration
after that. The test moves shard 3 from GROUPID4 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID4 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_before_move_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_move_db1_t1, 20)
self.assertEqual(row_cnt_shard_before_move_db2_t2, 20)
packet = self.proxy.sharding.move_shard("3", "GROUPID6")
self.check_xmlrpc_command_success(packet, None, False)
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_after_move_db1_t1,
row_cnt_shard_before_move_db1_t1)
self.assertEqual(row_cnt_shard_after_move_db2_t2,
row_cnt_shard_before_move_db2_t2)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the third shard's
#original group.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertEqual(global_table_count, 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard
#new group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_4(self):
'''Test the move of shard 4 and the global server configuration
after that. The test moves shard 4 from GROUPID5 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID5 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_before_move_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_move_db1_t1, 40)
self.assertEqual(row_cnt_shard_before_move_db2_t2, 40)
packet = self.proxy.sharding.move_shard("4", "GROUPID6")
self.check_xmlrpc_command_success(packet, None, False)
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_after_move_db1_t1,
row_cnt_shard_before_move_db1_t1)
self.assertEqual(row_cnt_shard_after_move_db2_t2,
row_cnt_shard_before_move_db2_t2)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the fourth shard's
#original group.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertEqual(global_table_count, 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard's new
#group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestShardingServices(unittest.TestCase):
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Configure the existing environment
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 2)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 3)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(2, "db2.t2", "userID2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(3, "db3.t3", "userID3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(1, "GROUPID2,GROUPID3",
"ENABLED")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
status = self.proxy.sharding.add_shard(2, "GROUPID4/0,GROUPID5/101",
"ENABLED")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
status = self.proxy.sharding.add_shard(3, "GROUPID6", "ENABLED")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
def test_list_shard_defns(self):
expected_shard_mapping_list1 = [
[1, "HASH", "GROUPID1"],
[2, "RANGE", "GROUPID1"],
[3, "HASH", "GROUPID1"]
]
status = self.proxy.sharding.list_definitions()
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_shard_mapping_list = status[2]
self.assertEqual(set(expected_shard_mapping_list1[0]),
set(obtained_shard_mapping_list[0]))
self.assertEqual(set(expected_shard_mapping_list1[1]),
set(obtained_shard_mapping_list[1]))
self.assertEqual(set(expected_shard_mapping_list1[2]),
set(obtained_shard_mapping_list[2]))
def test_list_shard_mappings(self):
status = self.proxy.sharding.list_tables("HASH")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], [
{"shard_mapping_id":1,
"table_name":"db1.t1",
"column_name":"userID1",
"type_name":"HASH",
"global_group":"GROUPID1"},
{"shard_mapping_id":3,
"table_name":"db3.t3",
"column_name":"userID3",
"type_name":"HASH",
"global_group":"GROUPID1"}
])
status = self.proxy.sharding.list_tables("RANGE")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], [
{"shard_mapping_id":2,
"table_name":"db2.t2",
"column_name":"userID2",
"type_name":"RANGE",
"global_group":"GROUPID1"}
])
def test_shard_prune(self):
status = self.proxy.sharding.prune_shard("db2.t2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
status = self.proxy.sharding.lookup_servers("db2.t2", 1, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MAX(userID2) FROM db2.t2",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MIN(userID2) FROM db2.t2",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 1)
status = self.proxy.sharding.lookup_servers("db2.t2", 101, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MAX(userID2) FROM db2.t2",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt(
"SELECT MIN(userID2) FROM db2.t2",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 101)
def tearDown(self):
"""Clean up the existing environment
"""
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.remove_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.remove_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.remove_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.remove_shard(4)
self.proxy.sharding.disable_shard(5)
self.proxy.sharding.remove_shard(5)
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestSharding(unittest.TestCase):
def setUp(self):
"""Configure the existing environment
"""
self.__options_1 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_1.mysql.com:3060",
}
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa75a12a-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_2.mysql.com:3060",
}
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
self.__group_1.add_server(self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS prune_db")
self.__server_3.exec_stmt("CREATE DATABASE prune_db")
self.__server_3.exec_stmt("CREATE TABLE prune_db.prune_table"
"(userID INT, name VARCHAR(30))")
self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(101, 'TEST 1')")
self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(202, 'TEST 2')")
self.__options_4 = {
"uuid" : _uuid.UUID("{dd75a12a-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_4.mysql.com:3060",
}
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_3)
self.__group_2.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_3)
self.__options_5 = {
"uuid" : _uuid.UUID("{ee75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS prune_db")
self.__server_5.exec_stmt("CREATE DATABASE prune_db")
self.__server_5.exec_stmt("CREATE TABLE prune_db.prune_table"
"(userID INT, name VARCHAR(30))")
self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(101, 'TEST 1')")
self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(202, 'TEST 2')")
self.__options_6 = {
"uuid" : _uuid.UUID("{ff75a12a-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_6.mysql.com:3060",
}
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_5)
self.__group_3.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_5)
group_4 = Group("GROUPID4", "4TH description.")
Group.add(group_4)
group_5 = Group("GROUPID5", "5TH description.")
Group.add(group_5)
group_6 = Group("GROUPID6", "6TH description.")
Group.add(group_6)
group_7 = Group("GROUPID7", "7TH description.")
Group.add(group_7)
group_8 = Group("GROUPID8", "8TH description.")
Group.add(group_8)
group_9 = Group("GROUPID9", "9TH description.")
Group.add(group_9)
group_10 = Group("GROUPID10", "10TH description.")
Group.add(group_10)
group_11 = Group("GROUPID11", "11TH description.")
Group.add(group_11)
group_12 = Group("GROUPID12", "12TH description.")
Group.add(group_12)
group_13 = Group("GROUPID13", "13TH description.")
Group.add(group_13)
group_14 = Group("GROUPID14", "14TH description.")
Group.add(group_14)
self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEquals( self.__shard_mapping_list, [])
self.__shard_mapping_id_1 = ShardMapping.define("RANGE", "GROUPID10")
self.__shard_mapping_id_2 = ShardMapping.define("RANGE", "GROUPID11")
self.__shard_mapping_id_3 = ShardMapping.define("RANGE", "GROUPID12")
#Test with sharding type values in lower case
self.__shard_mapping_id_4 = ShardMapping.define("range", "GROUPID13")
self.__shard_mapping_id_5 = ShardMapping.define("range", "GROUPID14")
self.__shard_mapping_1 = \
ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID1")
self.__shard_mapping_2 = \
ShardMapping.add(self.__shard_mapping_id_2, "db2.t2", "userID2")
self.__shard_mapping_3 = \
ShardMapping.add(self.__shard_mapping_id_3, "db3.t3", "userID3")
self.__shard_mapping_4 = \
ShardMapping.add(self.__shard_mapping_id_4, "db4.t4", "userID4")
self.__shard_mapping_5 = \
ShardMapping.add(self.__shard_mapping_id_5, "prune_db.prune_table",
"userID")
self.__shard_id_1 = Shards.add("GROUPID1")
self.__shard_id_2 = Shards.add("GROUPID10")
self.__shard_id_3 = Shards.add("GROUPID11")
self.__shard_id_4 = Shards.add("GROUPID4")
self.__shard_id_5 = Shards.add("GROUPID5")
self.__shard_id_6 = Shards.add("GROUPID6")
self.__shard_id_7 = Shards.add("GROUPID7")
self.__shard_id_8 = Shards.add("GROUPID8")
self.__shard_id_9 = Shards.add("GROUPID9")
self.__shard_id_10 = Shards.add("GROUPID2")
self.__shard_id_11 = Shards.add("GROUPID3")
self.__range_sharding_specification_1 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
0,
self.__shard_id_1.shard_id
)
self.__range_sharding_specification_2 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
1001,
self.__shard_id_2.shard_id
)
self.__range_sharding_specification_3 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
2001,
self.__shard_id_3.shard_id
)
self.__range_sharding_specification_4 = RangeShardingSpecification.add(
self.__shard_mapping_2.shard_mapping_id,
3001,
self.__shard_id_4.shard_id
)
self.__range_sharding_specification_5 = RangeShardingSpecification.add(
self.__shard_mapping_2.shard_mapping_id,
4001,
self.__shard_id_5.shard_id
)
self.__range_sharding_specification_6 = RangeShardingSpecification.add(
self.__shard_mapping_3.shard_mapping_id,
6001,
self.__shard_id_6.shard_id
)
self.__range_sharding_specification_7 = RangeShardingSpecification.add(
self.__shard_mapping_3.shard_mapping_id,
7001,
self.__shard_id_7.shard_id
)
self.__range_sharding_specification_8 = RangeShardingSpecification.add(
self.__shard_mapping_4.shard_mapping_id,
8001,
self.__shard_id_8.shard_id
)
self.__range_sharding_specification_9 = RangeShardingSpecification.add(
self.__shard_mapping_4.shard_mapping_id,
10001,
self.__shard_id_9.shard_id
)
self.__range_sharding_specification_10 = RangeShardingSpecification.add(
self.__shard_mapping_5.shard_mapping_id,
100,
self.__shard_id_10.shard_id)
self.__range_sharding_specification_11 = RangeShardingSpecification.add(
self.__shard_mapping_5.shard_mapping_id,
201,
self.__shard_id_11.shard_id)
def tearDown(self):
"""Clean up the existing environment
"""
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS prune_db")
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS prune_db")
tests.utils.cleanup_environment()
def test_fetch_shard_mapping(self):
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_2 = ShardMapping.fetch("db2.t2")
shard_mapping_3 = ShardMapping.fetch("db3.t3")
shard_mapping_4 = ShardMapping.fetch("db4.t4")
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_1, shard_mapping_1))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_2, shard_mapping_2))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_3, shard_mapping_3))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_4, shard_mapping_4))
def test_fetch_sharding_scheme(self):
range_sharding_specifications = RangeShardingSpecification.list(1)
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[0],
self.__range_sharding_specification_1))
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[1],
self.__range_sharding_specification_2))
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[2],
self.__range_sharding_specification_3))
def test_lookup_sharding_scheme(self):
r_spec_1 = \
RangeShardingSpecification.lookup(500, self.__shard_mapping_id_1,
"RANGE")
self.assertEqual(r_spec_1.shard_id, self.__shard_id_1.shard_id)
r_spec_2 = \
RangeShardingSpecification.lookup(3500, self.__shard_mapping_id_2,
"RANGE")
self.assertEqual(r_spec_2.shard_id, self.__shard_id_4.shard_id)
r_spec_3 = \
RangeShardingSpecification.lookup(6500, self.__shard_mapping_id_3,
"RANGE")
self.assertEqual(r_spec_3.shard_id, self.__shard_id_6.shard_id)
def test_shard_mapping_list_mappings(self):
shard_mappings = ShardMapping.list("RANGE")
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_1, shard_mappings[0]))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_2, shard_mappings[1]))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_3, shard_mappings[2]))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_4, shard_mappings[3]))
def test_shard_mapping_getters(self):
self.assertEqual(self.__shard_mapping_1.table_name, "db1.t1")
self.assertEqual(self.__shard_mapping_1.column_name, "userID1")
self.assertEqual(self.__shard_mapping_1.type_name, "RANGE")
self.assertEqual(self.__shard_mapping_1.shard_mapping_id, 1)
self.assertEqual(self.__shard_mapping_1.global_group, "GROUPID10")
def test_shard_mapping_remove(self):
self.__range_sharding_specification_1.remove()
self.__range_sharding_specification_2.remove()
self.__range_sharding_specification_3.remove()
self.__range_sharding_specification_4.remove()
self.__range_sharding_specification_5.remove()
self.__range_sharding_specification_6.remove()
self.__range_sharding_specification_7.remove()
self.__range_sharding_specification_8.remove()
self.__range_sharding_specification_9.remove()
self.__range_sharding_specification_10.remove()
self.__shard_id_1.remove()
self.__shard_id_2.remove()
self.__shard_id_3.remove()
self.__shard_id_4.remove()
self.__shard_id_5.remove()
self.__shard_id_6.remove()
self.__shard_id_7.remove()
self.__shard_id_8.remove()
self.__shard_id_9.remove()
self.__shard_id_10.remove()
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_1.remove()
def test_range_sharding_specification_getters(self):
self.assertEqual(
self.__range_sharding_specification_1.shard_mapping_id, 1)
self.assertEqual(self.__range_sharding_specification_1.lower_bound, 0)
self.assertEqual(self.__range_sharding_specification_1.shard_id, 1)
def test_list_shard_mapping(self):
expected_shard_mapping_list1 = [1, "RANGE", "GROUPID10"]
expected_shard_mapping_list2 = [2, "RANGE", "GROUPID11"]
expected_shard_mapping_list3 = [3, "RANGE", "GROUPID12"]
expected_shard_mapping_list4 = [4, "RANGE", "GROUPID13"]
expected_shard_mapping_list5 = [5, "RANGE", "GROUPID14"]
obtained_shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEqual(
set(expected_shard_mapping_list1),
set(obtained_shard_mapping_list[0])
)
self.assertEqual(
set(expected_shard_mapping_list2),
set(obtained_shard_mapping_list[1])
)
self.assertEqual(
set(expected_shard_mapping_list3),
set(obtained_shard_mapping_list[2])
)
self.assertEqual(
set(expected_shard_mapping_list4),
set(obtained_shard_mapping_list[3])
)
self.assertEqual(
set(expected_shard_mapping_list5),
set(obtained_shard_mapping_list[4])
)
class TestHashMoveGlobal(unittest.TestCase):
"""Contains unit tests for testing the shard move operation and for
verifying that the global server configuration remains constant after
the shard move configuration.
"""
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Configure the existing environment
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(
1,
"GROUPID2,GROUPID3,GROUPID4,GROUPID5",
"ENABLED"
)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
status = self.proxy.sharding.prune_shard("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
def test_move_shard_1(self):
'''Test the move of shard 1 and the global server configuration
after that. The test moves shard 1 from GROUPID2 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID2 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("1", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_setup_resharding_switch).")
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_2(self):
'''Test the move of shard 2 and the global server configuration
after that. The test moves shard 2 from GROUPID3 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID3 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("2", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_setup_resharding_switch).")
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_3(self):
'''Test the move of shard 3 and the global server configuration
after that. The test moves shard 3 from GROUPID4 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID4 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("3", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_setup_resharding_switch).")
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the third shard's
#original group.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard
#new group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_4(self):
'''Test the move of shard 4 and the global server configuration
after that. The test moves shard 4 from GROUPID5 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID5 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("4", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_setup_resharding_switch).")
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the fourth shard's
#original group.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard's new
#group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def tearDown(self):
"""Clean up the existing environment
"""
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.remove_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.remove_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.remove_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.remove_shard(4)
self.proxy.sharding.disable_shard(5)
self.proxy.sharding.remove_shard(5)
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestShardingServices(unittest.TestCase):
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Configure the existing environment
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(1, "GROUPID2,GROUPID3,"
"GROUPID4,GROUPID5,GROUPID6", "ENABLED")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
def tearDown(self):
"""Clean up the existing environment
"""
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.remove_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.remove_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.remove_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.remove_shard(4)
self.proxy.sharding.disable_shard(5)
self.proxy.sharding.remove_shard(5)
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
def test_add_shard_cannot_add_when_shards_exist_exception(self):
status = self.proxy.sharding.add_shard(1, "NON_EXISTENT_GROUP",
"ENABLED")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_add_shard).")
def test_add_shard_invalid_group_exception(self):
#Remove all the existing shards in the system. Since add shard
#shard will not be allowed when shards exist in the system
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.remove_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.remove_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.remove_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.remove_shard(4)
self.proxy.sharding.disable_shard(5)
self.proxy.sharding.remove_shard(5)
#Use an invalid group ID (WRONG_GROUP)
status = self.proxy.sharding.add_shard(4, "WRONG_GROUP", "ENABLED")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_add_shard).")
def test_add_shard_invalid_state_exception(self):
#Remove all the existing shards in the system. Since add shard
#shard will not be allowed when shards exist in the system
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.remove_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.remove_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.remove_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.remove_shard(4)
self.proxy.sharding.disable_shard(5)
self.proxy.sharding.remove_shard(5)
#WRONG_STATE is an invalid description of the shard state.
status = self.proxy.sharding.add_shard(4, "GROUP4", "WRONG_STATE")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_add_shard).")
def test_add_shard_invalid_shard_mapping(self):
status = self.proxy.sharding.add_shard(25000, "GROUPID4", "ENABLED")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_add_shard).")
def test_remove_shard_mapping_shards_exist(self):
#It is not an error to remove tables from a shard mapping.
status = self.proxy.sharding.remove_table("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard_mapping).")
def test_remove_sharding_specification(self):
status = self.proxy.sharding.disable_shard(1)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.remove_shard(1)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.disable_shard(2)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.remove_shard(2)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.disable_shard(3)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.remove_shard(3)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.disable_shard(4)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.remove_shard(4)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.disable_shard(5)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.remove_shard(5)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
self.assertEqual(status[0], False)
self.assertNotEqual(status[1], "")
self.assertEqual(status[2], True)
def test_remove_sharding_specification_exception(self):
status = self.proxy.sharding.remove_shard(1)
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_remove_shard).")
def test_lookup_shard_mapping(self):
status = self.proxy.sharding.lookup_table("db1.t1")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], {"shard_mapping_id":1,
"table_name":"db1.t1",
"column_name":"userID1",
"type_name":"HASH",
"global_group":"GROUPID1"})
def test_list(self):
status = self.proxy.sharding.list_tables("HASH")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], [{"shard_mapping_id":1,
"table_name":"db1.t1",
"column_name":"userID1",
"type_name":"HASH",
"global_group":"GROUPID1"}])
def test_list_exception(self):
status = self.proxy.sharding.list_tables("Wrong")
self.assertEqual(status[0], False)
self.assertNotEqual(status[1], "")
self.assertEqual(status[2], True)
def test_lookup(self):
shard_1_hit_count = 0
shard_2_hit_count = 0
shard_3_hit_count = 0
shard_4_hit_count = 0
shard_5_hit_count = 0
for i in range(1, 200):
status = self.proxy.sharding.lookup_servers("db1.t1", i, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
if obtained_server_list[0][1] == self.__server_2.address and \
str(obtained_server_list[0][0]) == str(self.__server_2.uuid):
shard_1_hit_count = shard_1_hit_count + 1
elif obtained_server_list[0][1] == self.__server_3.address and \
str(obtained_server_list[0][0]) == str(self.__server_3.uuid):
shard_2_hit_count = shard_2_hit_count + 1
elif obtained_server_list[0][1] == self.__server_4.address and \
str(obtained_server_list[0][0]) == str(self.__server_4.uuid):
shard_3_hit_count = shard_3_hit_count + 1
elif obtained_server_list[0][1] == self.__server_5.address and \
str(obtained_server_list[0][0]) == str(self.__server_5.uuid):
shard_4_hit_count = shard_4_hit_count + 1
elif obtained_server_list[0][1] == self.__server_6.address and \
str(obtained_server_list[0][0]) == str(self.__server_6.uuid):
shard_5_hit_count = shard_5_hit_count + 1
self.assertTrue(shard_1_hit_count > 0)
self.assertTrue(shard_2_hit_count > 0)
self.assertTrue(shard_3_hit_count > 0)
self.assertTrue(shard_4_hit_count > 0)
self.assertTrue(shard_5_hit_count > 0)
def test_lookup_disabled_exception(self):
#Since inhashing it is difficult to guess which shard the value will
#fall into, disable both the shards.
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.disable_shard(5)
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
self.assertEqual(status[0], False)
self.assertNotEqual(status[1], "")
self.assertEqual(status[2], True)
def test_lookup_wrong_table_exception(self):
status = self.proxy.sharding.lookup_servers("Wrong", 500, "LOCAL")
self.assertEqual(status[0], False)
self.assertNotEqual(status[1], "")
self.assertEqual(status[2], True)
def test_list_shard_mappings(self):
expected_shard_mapping_list1 = [1, "HASH", "GROUPID1"]
status = self.proxy.sharding.list_definitions()
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_shard_mapping_list = status[2]
self.assertEqual(
set(expected_shard_mapping_list1),
set(obtained_shard_mapping_list[0])
)
def test_enable_shard_exception(self):
status = self.proxy.sharding.enable_shard(25000)
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_enable_shard).")
def test_disable_shard_exception(self):
status = self.proxy.sharding.disable_shard(25000)
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_disable_shard).")
