class TestMySQLMaster(tests.utils.TestCase):
"""Unit test for the configuration file handling.
"""
def setUp(self):
"""Configure the existing environment
"""
uuid = MySQLServer.discover_uuid(OPTIONS_MASTER["address"])
OPTIONS_MASTER["uuid"] = _uuid.UUID(uuid)
self.master = MySQLServer(**OPTIONS_MASTER)
self.master.connect()
reset_master(self.master)
self.master.read_only = True
self.master.read_only = False
def tearDown(self):
"""Clean up the existing environment
"""
cleanup_environment()
self.master.disconnect()
def test_master_binary_log(self):
"""Test the get_master_status() function.
"""
# Note this is only being tested with the binary log.
master = self.master
# Get master status.
check = re.compile('\w+-bin.000001')
ret = get_master_status(master)
self.assertNotEqual(check.match(ret[0][0]), None)
# Reset Master.
reset_master(master)
ret = get_master_status(master)
self.assertTrue(ret[0][1] in (151, 154)) # Format descriptor event.
def test_master_health(self):
"""Test the check_master_issues() function.
"""
# Note this is only being tested with the binary log.
master = self.master
# Check health as a master before calling connect.
master.disconnect()
error, result = check_master_issues(master)
expected_result = {
'is_gtid_not_enabled': False,
'is_slave_updates_not_enabled': False,
'is_not_running': True,
'is_binlog_not_enabled': False,
'no_rpl_user': False
}
self.assertEqual(error, True)
self.assertEqual(result, expected_result)
# Check health as a master after calling connect.
master.connect()
error, _ = check_master_issues(master)
self.assertEqual(error, False)
def test_max_connections(self):
uuid = MySQLServer.discover_uuid(OPTIONS["address"])
server = MySQLServer(
_uuid.UUID(uuid), OPTIONS["address"],
)
server.connect()
res = server.get_variable("max_connections")
self.assertNotEqual(int(res), 0)
def test_max_connections(self):
uuid = MySQLServer.discover_uuid(OPTIONS["address"])
server = MySQLServer(
_uuid.UUID(uuid),
OPTIONS["address"],
)
server.connect()
res = server.get_variable("max_connections")
self.assertNotEqual(int(res), 0)
class TestMySQLMaster(unittest.TestCase):
"""Unit test for the configuration file handling.
"""
def setUp(self):
"""Configure the existing environment
"""
uuid = MySQLServer.discover_uuid(OPTIONS_MASTER["address"])
OPTIONS_MASTER["uuid"] = _uuid.UUID(uuid)
self.master = MySQLServer(**OPTIONS_MASTER)
self.master.connect()
reset_master(self.master)
self.master.read_only = True
self.master.read_only = False
def tearDown(self):
"""Clean up the existing environment
"""
cleanup_environment()
self.master.disconnect()
def test_master_binary_log(self):
"""Test the get_master_status() function.
"""
# Note this is only being tested with the binary log.
master = self.master
# Get master status.
check = re.compile('\w+-bin.000001')
ret = get_master_status(master)
self.assertNotEqual(check.match(ret[0][0]), None)
# Reset Master.
reset_master(master)
ret = get_master_status(master)
self.assertEqual(int(ret[0][1]), 151) # Format descriptor event.
def test_master_health(self):
"""Test the check_master_issues() function.
"""
# Note this is only being tested with the binary log.
master = self.master
# Check health as a master before calling connect.
master.disconnect()
ret = check_master_issues(master)
self.assertEqual(ret, {'is_running': False})
# Check health as a master after calling connect.
master.connect()
ret = check_master_issues(master)
self.assertEqual(ret, {})
def configure_servers(options):
"""Check if some MySQL's addresses were specified and the number is
greater than NUMBER_OF_SERVERS.
"""
import tests.utils as _test_utils
from mysql.fabric.server import (
MySQLServer,
ConnectionPool,
)
try:
servers = _test_utils.MySQLInstances()
servers.state_store_address = "{host}:{port}".format(
host=options.host, port=options.port
)
servers.user = options.db_user
servers.passwd = None
servers.root_user = options.user
servers.root_passwd = options.password
if options.servers:
for address in options.servers.split():
servers.add_address(address)
uuid = MySQLServer.discover_uuid(
address=address, user=servers.root_user,
passwd=servers.root_passwd
)
server = MySQLServer(
_uuid.UUID(uuid), address=address, user=servers.root_user,
passwd=servers.root_passwd
)
server.connect()
server.set_session_binlog(False)
server.exec_stmt(
"GRANT {privileges} ON *.* TO '{user}'@'%%'".format(
privileges=", ".join(MySQLServer.ALL_PRIVILEGES),
user=servers.user)
)
server.exec_stmt("FLUSH PRIVILEGES")
server.set_session_binlog(True)
server.disconnect()
ConnectionPool().purge_connections(server.uuid)
if servers.get_number_addresses() < NUMBER_OF_SERVERS:
print "<<<<<<<<<< Some unit tests need %s MySQL Instances. " \
">>>>>>>>>> " % (NUMBER_OF_SERVERS, )
return False
except Exception as error:
print "Error configuring servers:", error
return False
return True
def test_connection_pool(self):
"""Test connection pool.
"""
# Configuration
uuid = MySQLServer.discover_uuid(OPTIONS["address"])
OPTIONS["uuid"] = uuid = _uuid.UUID(uuid)
# We need to use the server_user here, because only those
# connections are pooled.
options = OPTIONS.copy()
options["user"] = tests.utils.MySQLInstances().server_user
options["passwd"] = tests.utils.MySQLInstances().server_passwd
server_1 = MySQLServer(**options)
server_2 = MySQLServer(**options)
cnx_pool = ConnectionManager()
# Purge connections and check the number of connections in
# the pool.
cnx_pool.purge_connections(server_1)
self.assertEqual(cnx_pool.get_number_connections(server_1), 0)
# Connect and check the number of connections in the pool.
server_1.connect()
server_2.connect()
self.assertEqual(cnx_pool.get_number_connections(server_1), 0)
# Delete one of the servers and check the number of
# connections in the pool.
del server_1
server_1 = MySQLServer(**options)
self.assertEqual(cnx_pool.get_number_connections(server_1), 1)
# Delete one of the servers and check the number of
# connections in the pool.
del server_2
server_2 = MySQLServer(**options)
self.assertEqual(cnx_pool.get_number_connections(server_2), 2)
# Purge connections and check the number of connections in
# the pool. However, call purge_connections twice.
cnx_pool.purge_connections(server_1)
self.assertEqual(cnx_pool.get_number_connections(server_1), 0)
cnx_pool.purge_connections(server_2)
self.assertEqual(cnx_pool.get_number_connections(server_2), 0)
def test_connection_pool(self):
"""Test connection pool.
"""
# Configuration
uuid = MySQLServer.discover_uuid(OPTIONS["address"])
OPTIONS["uuid"] = uuid = _uuid.UUID(uuid)
server_1 = MySQLServer(**OPTIONS)
server_2 = MySQLServer(**OPTIONS)
cnx_pool = ConnectionPool()
# Purge connections and check the number of connections in
# the pool.
cnx_pool.purge_connections(uuid)
self.assertEqual(cnx_pool.get_number_connections(uuid), 0)
# Connect and check the number of connections in the pool.
server_1.connect()
server_2.connect()
self.assertEqual(cnx_pool.get_number_connections(uuid), 0)
# Delete one of the servers and check the number of
# connections in the pool.
del server_1
self.assertEqual(cnx_pool.get_number_connections(uuid), 1)
# Delete one of the servers and check the number of
# connections in the pool.
del server_2
self.assertEqual(cnx_pool.get_number_connections(uuid), 2)
# Purge connections and check the number of connections in
# the pool. However, call purge_connections twice.
cnx_pool.purge_connections(uuid)
self.assertEqual(cnx_pool.get_number_connections(uuid), 0)
cnx_pool.purge_connections(uuid)
self.assertEqual(cnx_pool.get_number_connections(uuid), 0)
class TestShardingServices(unittest.TestCase):
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Configure the existing environment
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_2.exec_stmt("CREATE DATABASE db3")
self.__server_2.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_2.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_3.exec_stmt("CREATE DATABASE db3")
self.__server_3.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_3.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_4.exec_stmt("CREATE DATABASE db3")
self.__server_4.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_4.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_5.exec_stmt("CREATE DATABASE db3")
self.__server_5.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_5.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_6.exec_stmt("CREATE DATABASE db2")
self.__server_6.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_6.exec_stmt("CREATE DATABASE db3")
self.__server_6.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_6.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(1, "db2.t2", "userID2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(1, "db3.t3", "userID3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(
1,
"GROUPID2,GROUPID3,GROUPID4,GROUPID5,GROUPID6",
"ENABLED"
)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
status = self.proxy.sharding.prune_shard("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
status = self.proxy.sharding.prune_shard("db2.t2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
status = self.proxy.sharding.prune_shard("db3.t3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
def tearDown(self):
"""Clean up the existing environment
"""
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.remove_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.remove_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.remove_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.remove_shard(4)
self.proxy.sharding.disable_shard(5)
self.proxy.sharding.remove_shard(5)
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
def test_prune_lookup_shard1(self):
'''Verify that after the prune the lookup of any pruned value in the
shard results in looking up the same shard.
'''
self.proxy.sharding.prune_shard("db1.t1")
rows = self.__server_2.exec_stmt("SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def test_prune_lookup_shard2(self):
'''Verify that after the prune the lookup of any pruned value in the
shard results in looking up the same shard.
'''
self.proxy.sharding.prune_shard("db2.t2")
rows = self.__server_2.exec_stmt("SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def test_prune_lookup_shard3(self):
'''Verify that after the prune the lookup of any pruned value in the
shard results in looking up the same shard.
'''
self.proxy.sharding.prune_shard("db3.t3")
rows = self.__server_2.exec_stmt("SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(
val,
1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def test_list_shard_mappings(self):
expected_shard_mapping_list1 = [1, "HASH", "GROUPID1"]
status = self.proxy.sharding.list_definitions()
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_shard_mapping_list = status[2]
self.assertEqual(set(expected_shard_mapping_list1),
set(obtained_shard_mapping_list[0]))
def test_lookup_shard_mapping(self):
status = self.proxy.sharding.lookup_table("db1.t1")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], {"shard_mapping_id":1,
"table_name":"db1.t1",
"column_name":"userID1",
"type_name":"HASH",
"global_group":"GROUPID1"})
status = self.proxy.sharding.lookup_table("db2.t2")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], {"shard_mapping_id":1,
"table_name":"db2.t2",
"column_name":"userID2",
"type_name":"HASH",
"global_group":"GROUPID1"})
class TestServerClone(unittest.TestCase):
"""Tests the mysqlfabric clone command to clone all the data in a server
registered in Fabric into another server. Create a Group and add some
servers. Insert data into one of the servers. Now clone this server into
the other servers in the group. Now start replication and ensure everything
works fine.
"""
def assertStatus(self, status, expect):
"""Asserts that the status obtained is similar to the status
expected.
"""
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Create the setup for performing the testing of the server clone
command.
"""
from __main__ import mysqldump_path, mysqlclient_path
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address":MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__options_3 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address":MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
#Insert data into Server 1
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_1.exec_stmt("CREATE DATABASE db1")
self.__server_1.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
self.__server_1.exec_stmt("INSERT INTO db1.t1 "
"VALUES(101, 'TEST 1')")
self.__server_1.exec_stmt("INSERT INTO db1.t1 "
"VALUES(202, 'TEST 2')")
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_1.exec_stmt("CREATE DATABASE db2")
self.__server_1.exec_stmt("CREATE TABLE db2.t1"
"(userID INT, name VARCHAR(30))")
self.__server_1.exec_stmt("INSERT INTO db2.t1 "
"VALUES(101, 'TEST 1')")
self.__server_1.exec_stmt("INSERT INTO db2.t1 "
"VALUES(202, 'TEST 2')")
def test_clone_from_group(self):
"""Verify the clone operation from a group.
"""
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
self.__group_1.add_server(self.__server_2)
self.__group_1.master = self.__server_2.uuid
try:
status = self.proxy.server.clone("GROUPID1", self.__server_2.address,
str(self.__server_1.uuid))
raise Exception("Cloning to a server inside Fabric should "
"throw a fault")
except:
pass
try:
status = self.proxy.server.clone("GROUPID1", self.__server_2.address,
uuid_server3)
raise Exception("Cloning from a server outside the "
"source group should throw a fault")
except:
pass
status = self.proxy.server.clone("GROUPID1", self.__server_3.address,
None)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_restore_server).")
rows = self.__server_3.exec_stmt(
"SELECT NAME FROM db1.t1",
{"fetch" : True})
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
rows = self.__server_3.exec_stmt(
"SELECT NAME FROM db2.t1",
{"fetch" : True})
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
def test_clone_error(self):
"""Verify that the clone operations throws an error
for wrong input values.
"""
try:
status = self.proxy.server.clone(None, self.__server_3.address,
str(self.__server_1.uuid))
raise Exception("Not providing a group "\
"should throw a fault")
except:
pass
def tearDown(self):
"""Clean up the existing environment
"""
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestShardingPrune(unittest.TestCase):
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Creates the following topology for testing,
GROUPID1 - localhost:13001, localhost:13002 - Global Group
GROUPID2 - localhost:13003, localhost:13004 - shard 1
GROUPID3 - localhost:13005, localhost:13006 - shard 2
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 601):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 601):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 601):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 601):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 601):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(
1,
"GROUPID2/1,GROUPID3/101,GROUPID4/201,"
"GROUPID5/301,GROUPID6/401",
"ENABLED"
)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
def test_prune_shard(self):
status = self.proxy.sharding.prune_shard("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
status = self.proxy.sharding.lookup_servers("db1.t1", 1, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MAX(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MIN(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 1)
status = self.proxy.sharding.lookup_servers("db1.t1", 101, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MAX(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt(
"SELECT MIN(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 101)
status = self.proxy.sharding.lookup_servers("db1.t1", 202, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MAX(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 300)
rows = shard_server.exec_stmt(
"SELECT MIN(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 201)
status = self.proxy.sharding.lookup_servers("db1.t1", 303, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MAX(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 400)
rows = shard_server.exec_stmt(
"SELECT MIN(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 301)
status = self.proxy.sharding.lookup_servers("db1.t1", 404, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt(
"SELECT MAX(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 600)
rows = shard_server.exec_stmt(
"SELECT MIN(userID) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 401)
def tearDown(self):
self.__server_2.exec_stmt("DROP TABLE db1.t1")
self.__server_2.exec_stmt("DROP DATABASE db1")
self.__server_3.exec_stmt("DROP TABLE db1.t1")
self.__server_3.exec_stmt("DROP DATABASE db1")
self.__server_4.exec_stmt("DROP TABLE db1.t1")
self.__server_4.exec_stmt("DROP DATABASE db1")
self.__server_5.exec_stmt("DROP TABLE db1.t1")
self.__server_5.exec_stmt("DROP DATABASE db1")
self.__server_6.exec_stmt("DROP TABLE db1.t1")
self.__server_6.exec_stmt("DROP DATABASE db1")
status = self.proxy.sharding.disable_shard("1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.disable_shard("2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.disable_shard("3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.disable_shard("4")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.disable_shard("5")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_disable_shard).")
status = self.proxy.sharding.remove_shard("1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.remove_shard("2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.remove_shard("3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.remove_shard("4")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.remove_shard("5")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard).")
status = self.proxy.sharding.remove_table("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard_mapping).")
status = self.proxy.sharding.remove_definition("1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard_mapping_defn).")
self.proxy.group.demote("GROUPID1")
self.proxy.group.demote("GROUPID2")
self.proxy.group.demote("GROUPID3")
self.proxy.group.demote("GROUPID4")
self.proxy.group.demote("GROUPID5")
self.proxy.group.demote("GROUPID6")
for group_id in ("GROUPID1", "GROUPID2", "GROUPID3",
"GROUPID4", "GROUPID5", "GROUPID6"):
status = self.proxy.group.lookup_servers(group_id)
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
status = \
self.proxy.group.remove(
group_id, obtained_server_list[0]["server_uuid"]
)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_server).")
status = self.proxy.group.destroy(group_id)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_destroy_group).")
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestHashSplitGlobal(unittest.TestCase):
"""Contains unit tests for testing the shard split operation and for
verifying that the global server configuration remains constant after
the shard split configuration.
"""
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Configure the existing environment
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_2.exec_stmt("CREATE DATABASE db3")
self.__server_2.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_3.exec_stmt("CREATE DATABASE db3")
self.__server_3.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_4.exec_stmt("CREATE DATABASE db3")
self.__server_4.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_5.exec_stmt("CREATE DATABASE db3")
self.__server_5.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(1, "db3.t3", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(1, "GROUPID2,GROUPID3,GROUPID4,GROUPID5", "ENABLED")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
status = self.proxy.sharding.prune_shard("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
status = self.proxy.sharding.prune_shard("db2.t2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
status = self.proxy.sharding.prune_shard("db3.t3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
def test_split_shard_1(self):
'''Test the split of shard 1 and the global server configuration
after that. The test splits shard 1 between GROUPID2 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_db3_t3 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("1", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
self.__server_6.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 ==
(row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b)
)
self.assertTrue(
row_cnt_shard_db2_t2 ==
(row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b)
)
self.assertTrue(
row_cnt_shard_db3_t3 ==
(row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b)
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_2(self):
'''Test the split of shard 2 and the global server configuration
after that. The test splits shard 2 between GROUPID3 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_db3_t3 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("2", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
self.__server_6.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 ==
(row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b)
)
self.assertTrue(
row_cnt_shard_db2_t2 ==
(row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b)
)
self.assertTrue(
row_cnt_shard_db3_t3 ==
(row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b)
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_3(self):
'''Test the split of shard 3 and the global server configuration
after that. The test splits shard 3 between GROUPID4 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_db3_t3 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("3", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
self.__server_6.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 ==
(row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b)
)
self.assertTrue(
row_cnt_shard_db2_t2 ==
(row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b)
)
self.assertTrue(
row_cnt_shard_db3_t3 ==
(row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b)
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_4(self):
'''Test the split of shard 4 and the global server configuration
after that. The test splits shard 4 between GROUPID5 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_db3_t3 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("4", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
self.__server_6.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3",
{"fetch" : True}
)
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 ==
(row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b)
)
self.assertTrue(
row_cnt_shard_db2_t2 ==
(row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b)
)
self.assertTrue(
row_cnt_shard_db3_t3 ==
(row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b)
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_hash_dump(self):
"""Test the dump of the HASH sharding information for the table.
"""
shardinginformation_1 = [0, 0, 0,
[['db1', 't1', 'userID',
'E2996A7B8509367020B55A4ACD2AE46A',
'1', 'HASH', 'GROUPID2', 'GROUPID1'],
['db1', 't1', 'userID',
'E88F547DF45C99F27646C76316FB21DF',
'2', 'HASH', 'GROUPID3', 'GROUPID1'],
['db1', 't1', 'userID',
'7A2E76448FF04233F3851A492BEF1090',
'3', 'HASH', 'GROUPID4', 'GROUPID1'],
['db1', 't1', 'userID',
'97427AA63E300F56536710F5D73A35FA',
'4', 'HASH', 'GROUPID5', 'GROUPID1']]]
self.assertEqual(
self.proxy.dump.sharding_information(0, "db1.t1"),
shardinginformation_1
)
def tearDown(self):
"""Clean up the existing environment
"""
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.remove_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.remove_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.remove_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.remove_shard(4)
self.proxy.sharding.disable_shard(5)
self.proxy.sharding.remove_shard(5)
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
def configure_servers(options, config):
"""Check if some MySQL's addresses were specified and the number is
greater than NUMBER_OF_SERVERS.
"""
import tests.utils as _test_utils
from mysql.fabric.server import (
MySQLServer,
ConnectionManager,
)
from mysql.fabric.backup import (
MySQLDump,
)
try:
servers = _test_utils.MySQLInstances()
# The administrative user as given in --user and --password options.
# In simple use cases "root" is used.
servers.user = options.user
servers.passwd = options.password
# Backing store - "fabric_store/storepw".
servers.state_store_address = config.get("storage", "address")
servers.store_user = config.get("storage", "user")
servers.store_passwd = config.get("storage", "password")
servers.store_db = config.get("storage", "database")
# Server user - "fabric_server/serverpw".
servers.server_user = config.get("servers", "user")
servers.server_passwd = config.get("servers", "password")
# Backup user - "fabric_backup/backuppw".
servers.backup_user = config.get("servers", "backup_user")
servers.backup_passwd = config.get("servers", "backup_password")
# Restore user - "fabric_restore/restorepw".
servers.restore_user = config.get("servers", "restore_user")
servers.restore_passwd = config.get("servers", "restore_password")
# Set up the backing store.
from mysql.fabric import persistence
uuid = MySQLServer.discover_uuid(
address=servers.state_store_address,
user=servers.user,
passwd=servers.passwd
)
server = MySQLServer(
_uuid.UUID(uuid), address=servers.state_store_address,
user=servers.user, passwd=servers.passwd
)
server.connect()
# Precautionary cleanup.
server.exec_stmt("DROP DATABASE IF EXISTS %s" % (servers.store_db,))
# Create store user.
_test_utils.create_test_user(
server,
servers.store_user,
servers.store_passwd,
[(persistence.required_privileges(),
"{db}.*".format(db=servers.store_db))]
)
# Set up managed servers.
if options.servers:
for address in options.servers.split():
servers.add_address(address)
uuid = MySQLServer.discover_uuid(
address=address,
user=servers.user,
passwd=servers.passwd
)
server = MySQLServer(
_uuid.UUID(uuid),
address=address,
user=servers.user,
passwd=servers.passwd
)
server.connect()
server.set_session_binlog(False)
server.read_only = False
# Drop user databases
server.set_foreign_key_checks(False)
databases = server.exec_stmt("SHOW DATABASES")
for database in databases:
if database[0] not in MySQLServer.NO_USER_DATABASES:
server.exec_stmt("DROP DATABASE IF EXISTS %s" %
(database[0],))
server.set_foreign_key_checks(True)
# Create server user.
_test_utils.create_test_user(
server,
servers.server_user,
servers.server_passwd,
[(MySQLServer.SERVER_PRIVILEGES, "*.*"),
(MySQLServer.SERVER_PRIVILEGES_DB, "mysql_fabric.*")]
)
# Create backup user.
_test_utils.create_test_user(
server,
servers.backup_user,
servers.backup_passwd,
[(MySQLDump.BACKUP_PRIVILEGES, "*.*")]
)
# Create restore user.
_test_utils.create_test_user(
server,
servers.restore_user,
servers.restore_passwd,
[(MySQLDump.RESTORE_PRIVILEGES, "*.*")]
)
server.set_session_binlog(True)
server.disconnect()
ConnectionManager().purge_connections(server)
if servers.get_number_addresses() < NUMBER_OF_SERVERS:
sys.stderr.write(
"<<<<<<<<<< Some unit tests need {0} MySQL " \
"Instances. >>>>>>>>>>\n".format(NUMBER_OF_SERVERS)
)
return False
except Exception as error:
sys.stderr.write(
"Error configuring servers: {0}\n".format(str(error))
)
import traceback
traceback.print_exc()
return False
return True
class TestHashSharding(unittest.TestCase):
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEquals( self.__shard_mapping_list, [])
self.__shard_mapping_id_1 = ShardMapping.define("HASH", "GROUPID1")
self.__shard_mapping_1 = ShardMapping.add(
self.__shard_mapping_id_1,
"db1.t1",
"userID"
)
self.__shard_1 = Shards.add("GROUPID2")
self.__shard_2 = Shards.add("GROUPID3")
self.__shard_3 = Shards.add("GROUPID4")
self.__shard_4 = Shards.add("GROUPID5")
self.__shard_5 = Shards.add("GROUPID6")
self.__hash_sharding_specification_1 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
self.__shard_1.shard_id
)
self.__hash_sharding_specification_2 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
self.__shard_2.shard_id
)
self.__hash_sharding_specification_3 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
self.__shard_3.shard_id
)
self.__hash_sharding_specification_4 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
self.__shard_4.shard_id
)
self.__hash_sharding_specification_5 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
self.__shard_5.shard_id
)
def test_hash_lookup(self):
"""Test the hash sharding lookup.
"""
shard_1_cnt = 0
shard_2_cnt = 0
shard_3_cnt = 0
shard_4_cnt = 0
shard_5_cnt = 0
#Lookup a range of keys to ensure that all the shards are
#utilized.
for i in range(0, 1000):
hash_sharding_spec_1 = HashShardingSpecification.lookup(
i,
self.__shard_mapping_id_1,
"HASH"
)
if self.__shard_1.shard_id == hash_sharding_spec_1.shard_id:
shard_1_cnt = shard_1_cnt + 1
elif self.__shard_2.shard_id == hash_sharding_spec_1.shard_id:
shard_2_cnt = shard_2_cnt + 1
elif self.__shard_3.shard_id == hash_sharding_spec_1.shard_id:
shard_3_cnt = shard_3_cnt + 1
elif self.__shard_4.shard_id == hash_sharding_spec_1.shard_id:
shard_4_cnt = shard_4_cnt + 1
elif self.__shard_5.shard_id == hash_sharding_spec_1.shard_id:
shard_5_cnt = shard_5_cnt + 1
#The following will ensure that both the hash shards are utilized
#to store the keys and the values are not skewed in one shard.
self.assertTrue(shard_1_cnt > 0)
self.assertTrue(shard_2_cnt > 0)
self.assertTrue(shard_3_cnt > 0)
self.assertTrue(shard_4_cnt > 0)
self.assertTrue(shard_5_cnt > 0)
def test_hash_remove(self):
"""Test the removal of hash shards.
"""
hash_sharding_specification_1 = HashShardingSpecification.fetch(1)
hash_sharding_specification_2 = HashShardingSpecification.fetch(2)
hash_sharding_specification_3 = HashShardingSpecification.fetch(3)
hash_sharding_specification_4 = HashShardingSpecification.fetch(4)
hash_sharding_specification_5 = HashShardingSpecification.fetch(5)
hash_sharding_specification_1.remove()
hash_sharding_specification_2.remove()
hash_sharding_specification_3.remove()
hash_sharding_specification_4.remove()
hash_sharding_specification_5.remove()
self.__shard_1.remove()
self.__shard_2.remove()
self.__shard_3.remove()
self.__shard_4.remove()
self.__shard_5.remove()
for i in range(0, 10):
hash_sharding_spec = HashShardingSpecification.lookup(
i,
self.__shard_mapping_id_1,
"HASH"
)
self.assertEqual(hash_sharding_spec, None)
def test_list_shard_mapping(self):
"""Test the listing of HASH shards in a shard mapping.
"""
expected_shard_mapping_list1 = [1, "HASH", "GROUPID1"]
obtained_shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEqual(set(expected_shard_mapping_list1),
set(obtained_shard_mapping_list[0]))
def test_shard_mapping_list_mappings(self):
"""Test the listing of all HASH shards in a shard mapping.
"""
shard_mappings = ShardMapping.list("HASH")
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_1, shard_mappings[0]))
def test_fetch_sharding_scheme(self):
"""Test the fetch method of the HASH sharding scheme.
"""
hash_sharding_specification_1 = HashShardingSpecification.fetch(1)
hash_sharding_specification_2 = HashShardingSpecification.fetch(2)
hash_sharding_specification_3 = HashShardingSpecification.fetch(3)
hash_sharding_specification_4 = HashShardingSpecification.fetch(4)
hash_sharding_specification_5 = HashShardingSpecification.fetch(5)
hash_sharding_specifications = HashShardingSpecification.list(1)
#list does not return the hashing specifications in order of shard_id,
#hence a direct comparison is not posssible.
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_1,
hash_sharding_specifications
)
)
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_2,
hash_sharding_specifications
)
)
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_3,
hash_sharding_specifications
)
)
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_4,
hash_sharding_specifications
)
)
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_5,
hash_sharding_specifications
)
)
def test_prune_shard(self):
rows = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 500)
status = self.proxy.sharding.prune_shard("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
rows = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
cnt1 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
cnt2 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
cnt3 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
cnt4 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True})
cnt5 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
self.assertTrue((cnt1 + cnt2 + cnt3 + cnt4 + cnt5) == 500)
def test_prune_lookup(self):
self.proxy.sharding.prune_shard("db1.t1")
rows = self.__server_2.exec_stmt(
"SELECT userID FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(
val,
self.__shard_mapping_id_1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(
val,
self.__shard_mapping_id_1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(
val,
self.__shard_mapping_id_1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(
val,
self.__shard_mapping_id_1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(
val,
self.__shard_mapping_id_1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def hash_sharding_specification_in_list(self,
hash_sharding_spec,
hash_sharding_specification_list
):
"""Verify if the given hash sharding specification is present in
the list of hash sharding specifications.
:param hash_sharding_spec: The hash sharding specification that
needs to be lookedup.
:param hash_sharding_specification_list: The list of hash sharding
specifications
:return: True if the hash sharding specification is present.
False otherwise.
"""
for i in range(0, len(hash_sharding_specification_list)):
if ShardingUtils.compare_hash_specifications(
hash_sharding_spec,
hash_sharding_specification_list[i]
):
return True
return False
def tearDown(self):
"""Tear down the state store setup.
"""
self.__server_2.exec_stmt("DROP TABLE db1.t1")
self.__server_2.exec_stmt("DROP DATABASE db1")
self.__server_3.exec_stmt("DROP TABLE db1.t1")
self.__server_3.exec_stmt("DROP DATABASE db1")
self.__server_4.exec_stmt("DROP TABLE db1.t1")
self.__server_4.exec_stmt("DROP DATABASE db1")
self.__server_5.exec_stmt("DROP TABLE db1.t1")
self.__server_5.exec_stmt("DROP DATABASE db1")
self.__server_6.exec_stmt("DROP TABLE db1.t1")
self.__server_6.exec_stmt("DROP DATABASE db1")
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestBackupMySQLDump(unittest.TestCase):
"""Test taking a backup from a source group to a destination group. The
source group handles the source shard and the destination group handles
the destination shard. The backup and restore helps take up a backup of
the source shard and setup the destination shard.
"""
def setUp(self):
"""Configure the existing environment
"""
from __main__ import mysqldump_path, mysqlclient_path
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address":MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS backup_db")
self.__server_1.exec_stmt("CREATE DATABASE backup_db")
self.__server_1.exec_stmt("CREATE TABLE backup_db.backup_table"
"(userID INT, name VARCHAR(30))")
self.__server_1.exec_stmt("CREATE TABLE backup_db.trigger_table"
"(count INT)")
self.__server_1.exec_stmt("INSERT INTO backup_db.trigger_table "
"VALUES(0)")
self.__server_1.exec_stmt("CREATE TRIGGER backup_db.backup_table_ai"
" AFTER INSERT ON backup_db.backup_table"
" FOR EACH ROW UPDATE backup_db.trigger_table"
" SET count = count + 1")
self.__server_1.exec_stmt("INSERT INTO backup_db.backup_table "
"VALUES(101, 'TEST 1')")
self.__server_1.exec_stmt("INSERT INTO backup_db.backup_table "
"VALUES(202, 'TEST 2')")
self.__server_1.exec_stmt("CREATE INDEX i1 ON"
" backup_db.backup_table (userID)")
self.__server_1.exec_stmt("CREATE EVENT backup_db.ev1"
" ON SCHEDULE EVERY 1 DAY DO SELECT 1")
self.mysqldump_path = mysqldump_path
self.mysqlclient_path = mysqlclient_path
def test_backup(self):
image = MySQLDump.backup(self.__server_1,
MySQLInstances().backup_user,
MySQLInstances().backup_passwd,
self.mysqldump_path)
MySQLDump.restore_fabric_server(self.__server_2,
MySQLInstances().restore_user,
MySQLInstances().restore_passwd,
image,
self.mysqlclient_path)
rows = self.__server_2.exec_stmt(
"SELECT NAME FROM backup_db.backup_table",
{"fetch" : True})
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestServerClone(tests.utils.TestCase):
"""Tests the mysqlfabric clone command to clone all the data in a server
registered in Fabric into another server. Create a Group and add some
servers. Insert data into one of the servers. Now clone this server into
the other servers in the group. Now start replication and ensure everything
works fine.
"""
def setUp(self):
"""Create the setup for performing the testing of the server clone
command.
"""
_LOGGER.critical("\n\nStart of test-fixture-SetUp(): %s\n" %
(self._testMethodName))
from __main__ import mysqldump_path, mysqlclient_path
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__options_3 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
#Insert data into Server 1
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_1.exec_stmt("CREATE DATABASE db1")
self.__server_1.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
self.__server_1.exec_stmt("INSERT INTO db1.t1 "
"VALUES(101, 'TEST 1')")
self.__server_1.exec_stmt("INSERT INTO db1.t1 "
"VALUES(202, 'TEST 2')")
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_1.exec_stmt("CREATE DATABASE db2")
self.__server_1.exec_stmt("CREATE TABLE db2.t1"
"(userID INT, name VARCHAR(30))")
self.__server_1.exec_stmt("INSERT INTO db2.t1 "
"VALUES(101, 'TEST 1')")
self.__server_1.exec_stmt("INSERT INTO db2.t1 "
"VALUES(202, 'TEST 2')")
_LOGGER.debug("End of test-fixture-SetUp(): %s\n" %
(self._testMethodName))
def test_clone_from_group(self):
"""Verify the clone operation from a group.
"""
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
self.__group_1.add_server(self.__server_2)
self.__group_1.master = self.__server_2.uuid
try:
status = self.proxy.server.clone("GROUPID1",
self.__server_2.address,
str(self.__server_1.uuid))
raise Exception("Cloning to a server inside Fabric should "
"throw a fault")
except:
pass
try:
status = self.proxy.server.clone("GROUPID1",
self.__server_2.address,
uuid_server3)
raise Exception("Cloning from a server outside the "
"source group should throw a fault")
except:
pass
status = self.proxy.server.clone("GROUPID1", self.__server_3.address,
None)
self.check_xmlrpc_command_result(status)
rows = self.__server_3.exec_stmt("SELECT NAME FROM db1.t1",
{"fetch": True})
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
rows = self.__server_3.exec_stmt("SELECT NAME FROM db2.t1",
{"fetch": True})
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
def test_clone_permissions(self):
"""Verify the permissions, needed for a clone operation.
Change password of the backup user, clone fails on backup.
Change password back,
Revoke SELECT from the backup user, clone fails on backup.
Grant SELECT back,
Change password of the restore user, clone fails on restore.
Change password back,
Revoke CREATE from the restore user, clone fails on restore.
Grant CREATE back.
Clone succeeds.
"""
#
# Skip test in trial-mode. Users are the same as admin user.
# Admin won't be able to grant a privilege back to itself.
#
if ((MySQLInstances().backup_user == MySQLInstances().user)
or (MySQLInstances().restore_user == MySQLInstances().user)):
# The trailing comma prevents a newline.
print "Skipping test_clone_permissions in trial-mode --- ",
return
server1 = self.__server_1
server2 = self.__server_2
server3 = self.__server_3
#
# Construct a group in the backing store.
#
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(server1)
self.__group_1.add_server(server2)
self.__group_1.master = server2.uuid
#
# Change password of the the backup user, try a clone, which
# fails, because the Fabric instance does still use the old
# password, change password back. This works, because Fabric
# does not pool the connections for backup and restore users. So
# they have to be freshly connected for backup and restore
# operations.
#
server1.exec_stmt(
"SET PASSWORD FOR '{user}'@'%' ="
" PASSWORD('foobar')".format(user=MySQLInstances().backup_user))
status = self.proxy.server.clone("GROUPID1", server3.address, None)
self.check_xmlrpc_command_result(status, has_error=True)
server1.exec_stmt("SET PASSWORD FOR '{user}'@'%' ="
" PASSWORD('{passwd}')".format(
user=MySQLInstances().backup_user,
passwd=MySQLInstances().backup_passwd))
#
# Revoke the SELECT privilege from the backup user,
# try a clone, which fails, grant SELECT back.
#
server1.exec_stmt("REVOKE SELECT ON *.* FROM '{user}'@'%'".format(
user=MySQLInstances().backup_user))
status = self.proxy.server.clone("GROUPID1", server3.address, None)
self.check_xmlrpc_command_result(status, has_error=True)
server1.exec_stmt("GRANT SELECT ON *.* TO '{user}'@'%'".format(
user=MySQLInstances().backup_user))
#
# Change password of the the restore user, try a clone, which
# fails, because the Fabric instance does still use the old
# password, change password back. This works, because Fabric
# does not pool the connections for backup and restore users. So
# they have to be freshly connected for backup and restore
# operations.
#
server3.exec_stmt(
"SET PASSWORD FOR '{user}'@'%' ="
" PASSWORD('foobar')".format(user=MySQLInstances().restore_user))
status = self.proxy.server.clone("GROUPID1", server3.address, None)
self.check_xmlrpc_command_result(status, has_error=True)
server3.exec_stmt("SET PASSWORD FOR '{user}'@'%' ="
" PASSWORD('{passwd}')".format(
user=MySQLInstances().restore_user,
passwd=MySQLInstances().restore_passwd))
#
# Revoke the CREATE privilege from the restore user,
# try a clone, which fails, grant CREATE back.
#
server3.exec_stmt("REVOKE CREATE ON *.* FROM '{user}'@'%'".format(
user=MySQLInstances().restore_user))
# We have to clear GTID_EXECUTED before we can restore.
reset_master(server3)
status = self.proxy.server.clone("GROUPID1", server3.address, None)
self.check_xmlrpc_command_result(status, has_error=True)
server3.exec_stmt("GRANT CREATE ON *.* TO '{user}'@'%'".format(
user=MySQLInstances().restore_user))
# We have to clear GTID_EXECUTED before we can try the next restore.
reset_master(server3)
#
# Run a successful clone.
#
status = self.proxy.server.clone("GROUPID1", server3.address, None)
self.check_xmlrpc_command_result(status)
rows = server3.exec_stmt("SELECT NAME FROM db1.t1", {"fetch": True})
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
rows = server3.exec_stmt("SELECT NAME FROM db2.t1", {"fetch": True})
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
def test_clone_error(self):
"""Verify that the clone operations throws an error
for wrong input values.
"""
try:
status = self.proxy.server.clone(None, self.__server_3.address,
str(self.__server_1.uuid))
raise Exception("Not providing a group "\
"should throw a fault")
except:
pass
def tearDown(self):
"""Clean up the existing environment
"""
_LOGGER.debug("Start of test-fixture-tearDown(): %s" %
(self._testMethodName))
tests.utils.cleanup_environment()
_LOGGER.debug("End of test-fixture-tearDown(): %s" %
(self._testMethodName))
class TestShardingServices(tests.utils.TestCase):
def setUp(self):
"""Configure the existing environment
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_2.exec_stmt("CREATE DATABASE db3")
self.__server_2.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_2.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_3.exec_stmt("CREATE DATABASE db3")
self.__server_3.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_3.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_4.exec_stmt("CREATE DATABASE db3")
self.__server_4.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_4.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_5.exec_stmt("CREATE DATABASE db3")
self.__server_5.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_5.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID1 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_6.exec_stmt("CREATE DATABASE db2")
self.__server_6.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(201, 401):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_6.exec_stmt("CREATE DATABASE db3")
self.__server_6.exec_stmt("CREATE TABLE db3.t3"
"(userID3 INT, name VARCHAR(30))")
for i in range(401, 601):
self.__server_6.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db2.t2", "userID2")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db3.t3", "userID3")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(
1,
"GROUPID2,GROUPID3,GROUPID4,GROUPID5,GROUPID6",
"ENABLED"
)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db2.t2")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db3.t3")
self.check_xmlrpc_command_result(status)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
def test_prune_lookup_shard1(self):
'''Verify that after the prune the lookup of any pruned value in the
shard results in looking up the same shard.
'''
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
rows = self.__server_2.exec_stmt("SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID1 FROM db1.t1",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def test_prune_lookup_shard2(self):
'''Verify that after the prune the lookup of any pruned value in the
shard results in looking up the same shard.
'''
status = self.proxy.sharding.prune_shard("db2.t2")
self.check_xmlrpc_command_result(status)
rows = self.__server_2.exec_stmt("SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID2 FROM db2.t2",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def test_prune_lookup_shard3(self):
'''Verify that after the prune the lookup of any pruned value in the
shard results in looking up the same shard.
'''
status = self.proxy.sharding.prune_shard("db3.t3")
self.check_xmlrpc_command_result(status)
rows = self.__server_2.exec_stmt("SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_1.shard_id,
1
)
rows = self.__server_3.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_2.shard_id,
2
)
rows = self.__server_4.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_3.shard_id,
3
)
rows = self.__server_5.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(
val,
1,
"HASH"
)
self.assertEqual(
hash_sharding_spec_4.shard_id,
4
)
rows = self.__server_6.exec_stmt(
"SELECT userID3 FROM db3.t3",
{"fetch" : True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1,
"HASH")
self.assertEqual(
hash_sharding_spec_5.shard_id,
5
)
def test_list_shard_mappings(self):
expected_shard_mapping = {
'mapping_id' : 1,
'type_name' : 'HASH',
'global_group_id' : 'GROUPID1'
}
status = self.proxy.sharding.list_definitions()
obtained_shard_mapping = self.check_xmlrpc_simple(status, {})
self.assertEqual(expected_shard_mapping, obtained_shard_mapping)
def test_lookup_shard_mapping(self):
expected_shard_mapping_1 = {
'mapping_id' : 1,
'table_name' : 'db1.t1',
'column_name' : 'userID1',
'type_name' : 'HASH',
'global_group' : 'GROUPID1'
}
status = self.proxy.sharding.lookup_table("db1.t1")
obtained_shard_mapping = self.check_xmlrpc_simple(status, {})
self.assertEqual(expected_shard_mapping_1, obtained_shard_mapping)
expected_shard_mapping_2 = {
'mapping_id' : 1,
'table_name' : 'db2.t2',
'column_name' : 'userID2',
'type_name' : 'HASH',
'global_group' : 'GROUPID1'
}
status = self.proxy.sharding.lookup_table("db2.t2")
obtained_shard_mapping = self.check_xmlrpc_simple(status, {})
self.assertEqual(expected_shard_mapping_2, obtained_shard_mapping)
class TestShardingPrune(tests.utils.TestCase):
"""Contains unit tests for testing the shard split operation and for
verifying that the global server configuration remains constant after
the shard split configuration.
"""
def setUp(self):
"""Creates the topology for testing.
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20) PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_6.exec_stmt("CREATE DATABASE db2")
self.__server_6.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 401):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10601):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100801):
self.__server_6.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("RANGE_STRING", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(
1,
"GROUPID2/a,GROUPID3/b,GROUPID4/c,GROUPID5/d,GROUPID6/e",
"ENABLED"
)
self.check_xmlrpc_command_result(status)
def test_prune_shard(self):
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", "a3", "LOCAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 70)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", "b12", "LOCAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 300)
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 300)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", "c35", "LOCAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 200)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", "d21", "LOCAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 600)
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 600)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", "e31", "LOCAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 800)
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch" : True}
)
self.assertTrue(int(rows[0][0]) == 800)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestHashSharding(tests.utils.TestCase):
def setUp(self):
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(3),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add(self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(4),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add(self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(5),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 501):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add(self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEquals(self.__shard_mapping_list, [])
self.__shard_mapping_id_1 = ShardMapping.define("HASH", "GROUPID1")
self.__shard_mapping_1 = ShardMapping.add(self.__shard_mapping_id_1,
"db1.t1", "userID")
self.__shard_1 = Shards.add("GROUPID2")
self.__shard_2 = Shards.add("GROUPID3")
self.__shard_3 = Shards.add("GROUPID4")
self.__shard_4 = Shards.add("GROUPID5")
self.__shard_5 = Shards.add("GROUPID6")
self.__hash_sharding_specification_1 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, self.__shard_1.shard_id)
self.__hash_sharding_specification_2 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, self.__shard_2.shard_id)
self.__hash_sharding_specification_3 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, self.__shard_3.shard_id)
self.__hash_sharding_specification_4 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, self.__shard_4.shard_id)
self.__hash_sharding_specification_5 = HashShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, self.__shard_5.shard_id)
def test_hash_lookup(self):
"""Test the hash sharding lookup.
"""
shard_1_cnt = 0
shard_2_cnt = 0
shard_3_cnt = 0
shard_4_cnt = 0
shard_5_cnt = 0
#Lookup a range of keys to ensure that all the shards are
#utilized.
for i in range(0, 1000):
hash_sharding_spec_1 = HashShardingSpecification.lookup(
i, self.__shard_mapping_id_1, "HASH")
if self.__shard_1.shard_id == hash_sharding_spec_1.shard_id:
shard_1_cnt = shard_1_cnt + 1
elif self.__shard_2.shard_id == hash_sharding_spec_1.shard_id:
shard_2_cnt = shard_2_cnt + 1
elif self.__shard_3.shard_id == hash_sharding_spec_1.shard_id:
shard_3_cnt = shard_3_cnt + 1
elif self.__shard_4.shard_id == hash_sharding_spec_1.shard_id:
shard_4_cnt = shard_4_cnt + 1
elif self.__shard_5.shard_id == hash_sharding_spec_1.shard_id:
shard_5_cnt = shard_5_cnt + 1
#The following will ensure that both the hash shards are utilized
#to store the keys and the values are not skewed in one shard.
self.assertTrue(shard_1_cnt > 0)
self.assertTrue(shard_2_cnt > 0)
self.assertTrue(shard_3_cnt > 0)
self.assertTrue(shard_4_cnt > 0)
self.assertTrue(shard_5_cnt > 0)
def test_hash_remove(self):
"""Test the removal of hash shards.
"""
hash_sharding_specification_1 = HashShardingSpecification.fetch(1)
hash_sharding_specification_2 = HashShardingSpecification.fetch(2)
hash_sharding_specification_3 = HashShardingSpecification.fetch(3)
hash_sharding_specification_4 = HashShardingSpecification.fetch(4)
hash_sharding_specification_5 = HashShardingSpecification.fetch(5)
hash_sharding_specification_1.remove()
hash_sharding_specification_2.remove()
hash_sharding_specification_3.remove()
hash_sharding_specification_4.remove()
hash_sharding_specification_5.remove()
self.__shard_1.remove()
self.__shard_2.remove()
self.__shard_3.remove()
self.__shard_4.remove()
self.__shard_5.remove()
for i in range(0, 10):
hash_sharding_spec = HashShardingSpecification.lookup(
i, self.__shard_mapping_id_1, "HASH")
self.assertEqual(hash_sharding_spec, None)
def test_list_shard_mapping(self):
"""Test the listing of HASH shards in a shard mapping.
"""
expected_shard_mapping_list1 = [1, "HASH", "GROUPID1"]
obtained_shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEqual(set(expected_shard_mapping_list1),
set(obtained_shard_mapping_list[0]))
def test_shard_mapping_list_mappings(self):
"""Test the listing of all HASH shards in a shard mapping.
"""
shard_mappings = ShardMapping.list("HASH")
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_1,
shard_mappings[0]))
def test_fetch_sharding_scheme(self):
"""Test the fetch method of the HASH sharding scheme.
"""
hash_sharding_specification_1 = HashShardingSpecification.fetch(1)
hash_sharding_specification_2 = HashShardingSpecification.fetch(2)
hash_sharding_specification_3 = HashShardingSpecification.fetch(3)
hash_sharding_specification_4 = HashShardingSpecification.fetch(4)
hash_sharding_specification_5 = HashShardingSpecification.fetch(5)
hash_sharding_specifications = HashShardingSpecification.list(1)
#list does not return the hashing specifications in order of shard_id,
#hence a direct comparison is not posssible.
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_1, hash_sharding_specifications))
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_2, hash_sharding_specifications))
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_3, hash_sharding_specifications))
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_4, hash_sharding_specifications))
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_5, hash_sharding_specifications))
def test_prune_shard(self):
rows = self.__server_2.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_3.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_4.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_5.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 500)
rows = self.__server_6.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 500)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
rows = self.__server_2.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
cnt1 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_3.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
cnt2 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_4.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
cnt3 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_5.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
cnt4 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
rows = self.__server_6.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
cnt5 = int(rows[0][0])
self.assertTrue(int(rows[0][0]) < 500)
self.assertTrue((cnt1 + cnt2 + cnt3 + cnt4 + cnt5) == 500)
def test_prune_lookup(self):
self.proxy.sharding.prune_shard("db1.t1")
rows = self.__server_2.exec_stmt("SELECT userID FROM db1.t1",
{"fetch": True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_1 = HashShardingSpecification.lookup(
val, self.__shard_mapping_id_1, "HASH")
self.assertEqual(hash_sharding_spec_1.shard_id, 1)
rows = self.__server_3.exec_stmt("SELECT userID FROM db1.t1",
{"fetch": True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_2 = HashShardingSpecification.lookup(
val, self.__shard_mapping_id_1, "HASH")
self.assertEqual(hash_sharding_spec_2.shard_id, 2)
rows = self.__server_4.exec_stmt("SELECT userID FROM db1.t1",
{"fetch": True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_3 = HashShardingSpecification.lookup(
val, self.__shard_mapping_id_1, "HASH")
self.assertEqual(hash_sharding_spec_3.shard_id, 3)
rows = self.__server_5.exec_stmt("SELECT userID FROM db1.t1",
{"fetch": True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_4 = HashShardingSpecification.lookup(
val, self.__shard_mapping_id_1, "HASH")
self.assertEqual(hash_sharding_spec_4.shard_id, 4)
rows = self.__server_6.exec_stmt("SELECT userID FROM db1.t1",
{"fetch": True})
for val in rows[0:len(rows)][0]:
hash_sharding_spec_5 = HashShardingSpecification.lookup(
val, self.__shard_mapping_id_1, "HASH")
self.assertEqual(hash_sharding_spec_5.shard_id, 5)
def hash_sharding_specification_in_list(self, hash_sharding_spec,
hash_sharding_specification_list):
"""Verify if the given hash sharding specification is present in
the list of hash sharding specifications.
:param hash_sharding_spec: The hash sharding specification that
needs to be lookedup.
:param hash_sharding_specification_list: The list of hash sharding
specifications
:return: True if the hash sharding specification is present.
False otherwise.
"""
for i in range(0, len(hash_sharding_specification_list)):
if ShardingUtils.compare_hash_specifications(
hash_sharding_spec, hash_sharding_specification_list[i]):
return True
return False
def tearDown(self):
"""Tear down the state store setup.
"""
tests.utils.cleanup_environment()
def test_privileges(self):
"""Test whether user's have the appropriate privileges.
"""
# Some privileges
MINIMUM_PRIVILEGES = [
"REPLICATION SLAVE", "REPLICATION CLIENT", "SUPER",
"SHOW DATABASES", "RELOAD"
]
# Connect to server as root and create temporary user.
uuid = MySQLServer.discover_uuid(OPTIONS["address"])
server = MySQLServer(
_uuid.UUID(uuid), OPTIONS["address"],
tests.utils.MySQLInstances().root_user,
tests.utils.MySQLInstances().root_passwd
)
ConnectionPool().purge_connections(_uuid.UUID(uuid))
server.connect()
server.set_session_binlog(False)
server.exec_stmt(
"CREATE USER 'jeffrey'@'%%' IDENTIFIED BY 'mypass'"
)
# Check if jeffrey (temporary user) has the appropriate privileges.
# There is not privilege associate to jeffrey.
new_server = MySQLServer(
_uuid.UUID(uuid), OPTIONS["address"], "jeffrey", "mypass"
)
new_server.connect()
self.assertFalse(
new_server.has_privileges(MINIMUM_PRIVILEGES)
)
# Check if jeffrey (temporary user) has the appropriate privileges.
# Grant required privileges except RELOAD
# There is no RELOAD on a global level.
privileges=", ".join([priv for priv in MINIMUM_PRIVILEGES
if priv != "RELOAD"]
)
server.exec_stmt(
"GRANT {privileges} ON *.* TO 'jeffrey'@'%%'".format(
privileges=privileges)
)
server.exec_stmt("FLUSH PRIVILEGES")
self.assertFalse(
new_server.has_privileges(MINIMUM_PRIVILEGES)
)
# Check if jeffrey (temporary user) has the appropriate privileges.
# The RELOAD on a global level was granted.
server.exec_stmt("GRANT RELOAD ON *.* TO 'jeffrey'@'%%'")
server.exec_stmt("FLUSH PRIVILEGES")
self.assertTrue(
new_server.has_privileges(MINIMUM_PRIVILEGES)
)
# Check if jeffrey (temporary user) has the appropriate privileges.
# Revoke privilegs from temporary user.
# There is no ALL on a global level.
server.exec_stmt("REVOKE ALL PRIVILEGES, GRANT OPTION FROM "
"'jeffrey'@'%%'"
)
server.exec_stmt("GRANT ALL ON fabric.* TO 'jeffrey'@'%%'")
server.exec_stmt("FLUSH PRIVILEGES")
self.assertFalse(
new_server.has_privileges(MINIMUM_PRIVILEGES)
)
# Check if jeffrey (temporary user) has the appropriate privileges.
# The ALL on a global level was granted.
server.exec_stmt("GRANT ALL ON *.* TO 'jeffrey'@'%%'")
server.exec_stmt("FLUSH PRIVILEGES")
self.assertTrue(
new_server.has_privileges(MINIMUM_PRIVILEGES)
)
# Drop temporary user.
server.exec_stmt("DROP USER 'jeffrey'@'%%'")
server.set_session_binlog(True)
server.disconnect()
new_server.disconnect()
ConnectionPool().purge_connections(_uuid.UUID(uuid))
class TestShardingPrune(unittest.TestCase):
"""Contains unit tests for testing the shard split operation and for
verifying that the global server configuration remains constant after
the shard split configuration.
"""
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Creates the topology for testing.
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(
1,
"GROUPID2/1,GROUPID3/101,GROUPID4/1001,GROUPID5/10001",
"ENABLED"
)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
status = self.proxy.sharding.prune_shard("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
def test_split_shard_1(self):
'''Test the split of shard 1 and the global server configuration
after that. The test splits shard 1 between GROUPID2 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_before_split_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_split_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_split_db1_t1 = \
int(row_cnt_shard_before_split_db1_t1[0][0])
row_cnt_shard_before_split_db2_t2 = \
int(row_cnt_shard_before_split_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_split_db1_t1, 70)
self.assertEqual(row_cnt_shard_before_split_db2_t2, 70)
status = self.proxy.sharding.split_shard("1", "GROUPID6", "36")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
row_cnt_shard_after_split_db1_t1_a = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_a = \
int(row_cnt_shard_after_split_db1_t1_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 35)
row_cnt_shard_after_split_db2_t2_a = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_a = \
int(row_cnt_shard_after_split_db2_t2_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 35)
row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_b = \
int(row_cnt_shard_after_split_db1_t1_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 35)
row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_b = \
int(row_cnt_shard_after_split_db2_t2_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 35)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_2(self):
'''Test the split of shard 2 and the global server configuration
after that. The test splits shard 2 between GROUPID3 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_before_split_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_split_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_split_db1_t1 = \
int(row_cnt_shard_before_split_db1_t1[0][0])
row_cnt_shard_before_split_db2_t2 = \
int(row_cnt_shard_before_split_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_split_db1_t1, 200)
self.assertEqual(row_cnt_shard_before_split_db2_t2, 200)
status = self.proxy.sharding.split_shard("2", "GROUPID6", "201")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
row_cnt_shard_after_split_db1_t1_a = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_a = \
int(row_cnt_shard_after_split_db1_t1_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 100)
row_cnt_shard_after_split_db2_t2_a = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_a = \
int(row_cnt_shard_after_split_db2_t2_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 100)
row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_b = \
int(row_cnt_shard_after_split_db1_t1_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 100)
row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_b = \
int(row_cnt_shard_after_split_db2_t2_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 100)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_3(self):
'''Test the split of shard 3 and the global server configuration
after that. The test splits shard 3 between GROUPID4 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_before_split_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_split_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_split_db1_t1 = \
int(row_cnt_shard_before_split_db1_t1[0][0])
row_cnt_shard_before_split_db2_t2 = \
int(row_cnt_shard_before_split_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_split_db1_t1, 200)
self.assertEqual(row_cnt_shard_before_split_db2_t2, 200)
status = self.proxy.sharding.split_shard("3", "GROUPID6", "1101")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
row_cnt_shard_after_split_db1_t1_a = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_a = \
int(row_cnt_shard_after_split_db1_t1_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 100)
row_cnt_shard_after_split_db2_t2_a = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_a = \
int(row_cnt_shard_after_split_db2_t2_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 100)
row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_b = \
int(row_cnt_shard_after_split_db1_t1_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 100)
row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_b = \
int(row_cnt_shard_after_split_db2_t2_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 100)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_4(self):
'''Test the split of shard 4 and the global server configuration
after that. The test splits shard 4 between GROUPID5 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_before_split_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_split_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_split_db1_t1 = \
int(row_cnt_shard_before_split_db1_t1[0][0])
row_cnt_shard_before_split_db2_t2 = \
int(row_cnt_shard_before_split_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_split_db1_t1, 400)
self.assertEqual(row_cnt_shard_before_split_db2_t2, 400)
status = self.proxy.sharding.split_shard("4", "GROUPID6", "100001")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
row_cnt_shard_after_split_db1_t1_a = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_a = \
int(row_cnt_shard_after_split_db1_t1_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 200)
row_cnt_shard_after_split_db2_t2_a = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_a = \
int(row_cnt_shard_after_split_db2_t2_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 200)
row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_b = \
int(row_cnt_shard_after_split_db1_t1_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 200)
row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_b = \
int(row_cnt_shard_after_split_db2_t2_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 200)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_error_1(self):
'''Test that splitting with an invalid split value fails to
split a shard.
'''
status = self.proxy.sharding.split_shard("3", "GROUPID6", "10")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_check_shard_information).")
def test_split_shard_error_2(self):
'''Test that splitting with an invalid split value fails to
split a shard.
'''
status = self.proxy.sharding.split_shard("3", "GROUPID6", "1101.5")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_check_shard_information).")
def test_split_shard_error_3(self):
'''Test that splitting with an invalid split value fails to
split a shard.
'''
self.__group_6.status = Group.INACTIVE
status = self.proxy.sharding.split_shard("3", "GROUPID6", "1101.5")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_check_shard_information).")
def test_split_shard_error_4(self):
'''Test that splitting with an empty split value fails to
split a shard.
'''
self.__group_6.status = Group.INACTIVE
status = self.proxy.sharding.split_shard("3", "GROUPID6", None)
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_check_shard_information).")
def tearDown(self):
status = self.proxy.sharding.disable_shard("1")
status = self.proxy.sharding.disable_shard("2")
status = self.proxy.sharding.disable_shard("3")
status = self.proxy.sharding.disable_shard("4")
status = self.proxy.sharding.disable_shard("5")
status = self.proxy.sharding.disable_shard("6")
status = self.proxy.sharding.remove_shard("1")
status = self.proxy.sharding.remove_shard("2")
status = self.proxy.sharding.remove_shard("3")
status = self.proxy.sharding.remove_shard("4")
status = self.proxy.sharding.remove_shard("5")
status = self.proxy.sharding.remove_shard("6")
status = self.proxy.sharding.remove_table("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard_mapping).")
status = self.proxy.sharding.remove_table("db2.t2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard_mapping).")
status = self.proxy.sharding.remove_definition("1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_shard_mapping_defn).")
self.proxy.group.demote("GROUPID1")
self.proxy.group.demote("GROUPID2")
self.proxy.group.demote("GROUPID3")
self.proxy.group.demote("GROUPID4")
self.proxy.group.demote("GROUPID5")
self.proxy.group.demote("GROUPID6")
for group_id in ("GROUPID1", "GROUPID2", "GROUPID3",
"GROUPID4", "GROUPID5", "GROUPID6"):
status = self.proxy.group.lookup_servers(group_id)
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
status = self.proxy.group.remove(
group_id, obtained_server_list[0]["server_uuid"]
)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_remove_server).")
status = self.proxy.group.destroy(group_id)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_destroy_group).")
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestSharding(unittest.TestCase):
def setUp(self):
"""Configure the existing environment
"""
self.__options_1 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": "server_1.mysql.com:3060",
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa75a12a-98d1-414c-96af-9e9d4b179678}"),
"address": "server_2.mysql.com:3060",
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
self.__group_1.add_server(self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS prune_db")
self.__server_3.exec_stmt("CREATE DATABASE prune_db")
self.__server_3.exec_stmt("CREATE TABLE prune_db.prune_table"
"(userID INT, name VARCHAR(30))")
self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(101, 'TEST 1')")
self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(202, 'TEST 2')")
self.__options_4 = {
"uuid": _uuid.UUID("{dd75a12a-98d1-414c-96af-9e9d4b179678}"),
"address": "server_4.mysql.com:3060",
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_3)
self.__group_2.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_3)
self.__options_5 = {
"uuid": _uuid.UUID("{ee75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS prune_db")
self.__server_5.exec_stmt("CREATE DATABASE prune_db")
self.__server_5.exec_stmt("CREATE TABLE prune_db.prune_table"
"(userID INT, name VARCHAR(30))")
self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(101, 'TEST 1')")
self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(202, 'TEST 2')")
self.__options_6 = {
"uuid": _uuid.UUID("{ff75a12a-98d1-414c-96af-9e9d4b179678}"),
"address": "server_6.mysql.com:3060",
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_5)
self.__group_3.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_5)
group_4 = Group("GROUPID4", "4TH description.")
Group.add(group_4)
group_5 = Group("GROUPID5", "5TH description.")
Group.add(group_5)
group_6 = Group("GROUPID6", "6TH description.")
Group.add(group_6)
group_7 = Group("GROUPID7", "7TH description.")
Group.add(group_7)
group_8 = Group("GROUPID8", "8TH description.")
Group.add(group_8)
group_9 = Group("GROUPID9", "9TH description.")
Group.add(group_9)
group_10 = Group("GROUPID10", "10TH description.")
Group.add(group_10)
group_11 = Group("GROUPID11", "11TH description.")
Group.add(group_11)
group_12 = Group("GROUPID12", "12TH description.")
Group.add(group_12)
group_13 = Group("GROUPID13", "13TH description.")
Group.add(group_13)
group_14 = Group("GROUPID14", "14TH description.")
Group.add(group_14)
self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEquals(self.__shard_mapping_list, [])
self.__shard_mapping_id_1 = ShardMapping.define("RANGE", "GROUPID10")
self.__shard_mapping_id_2 = ShardMapping.define("RANGE", "GROUPID11")
self.__shard_mapping_id_3 = ShardMapping.define("RANGE", "GROUPID12")
#Test with sharding type values in lower case
self.__shard_mapping_id_4 = ShardMapping.define("range", "GROUPID13")
self.__shard_mapping_id_5 = ShardMapping.define("range", "GROUPID14")
self.__shard_mapping_1 = \
ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID1")
self.__shard_mapping_2 = \
ShardMapping.add(self.__shard_mapping_id_2, "db2.t2", "userID2")
self.__shard_mapping_3 = \
ShardMapping.add(self.__shard_mapping_id_3, "db3.t3", "userID3")
self.__shard_mapping_4 = \
ShardMapping.add(self.__shard_mapping_id_4, "db4.t4", "userID4")
self.__shard_mapping_5 = \
ShardMapping.add(self.__shard_mapping_id_5, "prune_db.prune_table",
"userID")
self.__shard_id_1 = Shards.add("GROUPID1")
self.__shard_id_2 = Shards.add("GROUPID10")
self.__shard_id_3 = Shards.add("GROUPID11")
self.__shard_id_4 = Shards.add("GROUPID4")
self.__shard_id_5 = Shards.add("GROUPID5")
self.__shard_id_6 = Shards.add("GROUPID6")
self.__shard_id_7 = Shards.add("GROUPID7")
self.__shard_id_8 = Shards.add("GROUPID8")
self.__shard_id_9 = Shards.add("GROUPID9")
self.__shard_id_10 = Shards.add("GROUPID2")
self.__shard_id_11 = Shards.add("GROUPID3")
self.__range_sharding_specification_1 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, 0,
self.__shard_id_1.shard_id)
self.__range_sharding_specification_2 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, 1001,
self.__shard_id_2.shard_id)
self.__range_sharding_specification_3 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id, 2001,
self.__shard_id_3.shard_id)
self.__range_sharding_specification_4 = RangeShardingSpecification.add(
self.__shard_mapping_2.shard_mapping_id, 3001,
self.__shard_id_4.shard_id)
self.__range_sharding_specification_5 = RangeShardingSpecification.add(
self.__shard_mapping_2.shard_mapping_id, 4001,
self.__shard_id_5.shard_id)
self.__range_sharding_specification_6 = RangeShardingSpecification.add(
self.__shard_mapping_3.shard_mapping_id, 6001,
self.__shard_id_6.shard_id)
self.__range_sharding_specification_7 = RangeShardingSpecification.add(
self.__shard_mapping_3.shard_mapping_id, 7001,
self.__shard_id_7.shard_id)
self.__range_sharding_specification_8 = RangeShardingSpecification.add(
self.__shard_mapping_4.shard_mapping_id, 8001,
self.__shard_id_8.shard_id)
self.__range_sharding_specification_9 = RangeShardingSpecification.add(
self.__shard_mapping_4.shard_mapping_id, 10001,
self.__shard_id_9.shard_id)
self.__range_sharding_specification_10 = RangeShardingSpecification.add(
self.__shard_mapping_5.shard_mapping_id, 100,
self.__shard_id_10.shard_id)
self.__range_sharding_specification_11 = RangeShardingSpecification.add(
self.__shard_mapping_5.shard_mapping_id, 201,
self.__shard_id_11.shard_id)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
def test_fetch_shard_mapping(self):
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_2 = ShardMapping.fetch("db2.t2")
shard_mapping_3 = ShardMapping.fetch("db3.t3")
shard_mapping_4 = ShardMapping.fetch("db4.t4")
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_1,
shard_mapping_1))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_2,
shard_mapping_2))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_3,
shard_mapping_3))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_4,
shard_mapping_4))
def test_fetch_sharding_scheme(self):
range_sharding_specifications = RangeShardingSpecification.list(1)
self.assertTrue(
ShardingUtils.compare_range_specifications(
range_sharding_specifications[0],
self.__range_sharding_specification_1))
self.assertTrue(
ShardingUtils.compare_range_specifications(
range_sharding_specifications[1],
self.__range_sharding_specification_2))
self.assertTrue(
ShardingUtils.compare_range_specifications(
range_sharding_specifications[2],
self.__range_sharding_specification_3))
def test_lookup_sharding_scheme(self):
r_spec_1 = \
RangeShardingSpecification.lookup(500, self.__shard_mapping_id_1,
"RANGE")
self.assertEqual(r_spec_1.shard_id, self.__shard_id_1.shard_id)
r_spec_2 = \
RangeShardingSpecification.lookup(3500, self.__shard_mapping_id_2,
"RANGE")
self.assertEqual(r_spec_2.shard_id, self.__shard_id_4.shard_id)
r_spec_3 = \
RangeShardingSpecification.lookup(6500, self.__shard_mapping_id_3,
"RANGE")
self.assertEqual(r_spec_3.shard_id, self.__shard_id_6.shard_id)
def test_shard_mapping_list_mappings(self):
shard_mappings = ShardMapping.list("RANGE")
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_1,
shard_mappings[0]))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_2,
shard_mappings[1]))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_3,
shard_mappings[2]))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_4,
shard_mappings[3]))
def test_shard_mapping_getters(self):
self.assertEqual(self.__shard_mapping_1.table_name, "db1.t1")
self.assertEqual(self.__shard_mapping_1.column_name, "userID1")
self.assertEqual(self.__shard_mapping_1.type_name, "RANGE")
self.assertEqual(self.__shard_mapping_1.shard_mapping_id, 1)
self.assertEqual(self.__shard_mapping_1.global_group, "GROUPID10")
def test_shard_mapping_remove(self):
self.__range_sharding_specification_1.remove()
self.__range_sharding_specification_2.remove()
self.__range_sharding_specification_3.remove()
self.__range_sharding_specification_4.remove()
self.__range_sharding_specification_5.remove()
self.__range_sharding_specification_6.remove()
self.__range_sharding_specification_7.remove()
self.__range_sharding_specification_8.remove()
self.__range_sharding_specification_9.remove()
self.__range_sharding_specification_10.remove()
self.__shard_id_1.remove()
self.__shard_id_2.remove()
self.__shard_id_3.remove()
self.__shard_id_4.remove()
self.__shard_id_5.remove()
self.__shard_id_6.remove()
self.__shard_id_7.remove()
self.__shard_id_8.remove()
self.__shard_id_9.remove()
self.__shard_id_10.remove()
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_1.remove()
def test_range_sharding_specification_getters(self):
self.assertEqual(
self.__range_sharding_specification_1.shard_mapping_id, 1)
self.assertEqual(self.__range_sharding_specification_1.lower_bound, 0)
self.assertEqual(self.__range_sharding_specification_1.shard_id, 1)
def test_list_shard_mapping(self):
expected_shard_mapping_list1 = [1, "RANGE", "GROUPID10"]
expected_shard_mapping_list2 = [2, "RANGE", "GROUPID11"]
expected_shard_mapping_list3 = [3, "RANGE", "GROUPID12"]
expected_shard_mapping_list4 = [4, "RANGE", "GROUPID13"]
expected_shard_mapping_list5 = [5, "RANGE", "GROUPID14"]
obtained_shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEqual(set(expected_shard_mapping_list1),
set(obtained_shard_mapping_list[0]))
self.assertEqual(set(expected_shard_mapping_list2),
set(obtained_shard_mapping_list[1]))
self.assertEqual(set(expected_shard_mapping_list3),
set(obtained_shard_mapping_list[2]))
self.assertEqual(set(expected_shard_mapping_list4),
set(obtained_shard_mapping_list[3]))
self.assertEqual(set(expected_shard_mapping_list5),
set(obtained_shard_mapping_list[4]))
class TestHashSplitGlobal(tests.utils.TestCase):
"""Contains unit tests for testing the shard split operation and for
verifying that the global server configuration remains constant after
the shard split configuration.
"""
def setUp(self):
"""Configure the existing environment
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_2.exec_stmt("CREATE DATABASE db3")
self.__server_2.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_3.exec_stmt("CREATE DATABASE db3")
self.__server_3.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(3),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_4.exec_stmt("CREATE DATABASE db3")
self.__server_4.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add(self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(4),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db3")
self.__server_5.exec_stmt("CREATE DATABASE db3")
self.__server_5.exec_stmt("CREATE TABLE db3.t3"
"(userID INT, Department INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db3.t3 "
"VALUES(%s, %s)" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add(self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(5),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add(self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db3.t3", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(
1, "GROUPID2,GROUPID3,GROUPID4,GROUPID5", "ENABLED")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db2.t2")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db3.t3")
self.check_xmlrpc_command_result(status)
def test_split_shard_1(self):
'''Test the split of shard 1 and the global server configuration
after that. The test splits shard 1 between GROUPID2 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_db3_t3 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("1", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
self.__server_2.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b))
self.assertTrue(
row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b))
self.assertTrue(
row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b))
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_2(self):
'''Test the split of shard 2 and the global server configuration
after that. The test splits shard 2 between GROUPID3 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_db3_t3 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("2", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
self.__server_3.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b))
self.assertTrue(
row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b))
self.assertTrue(
row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b))
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_3(self):
'''Test the split of shard 3 and the global server configuration
after that. The test splits shard 3 between GROUPID4 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_db3_t3 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("3", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
self.__server_4.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b))
self.assertTrue(
row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b))
self.assertTrue(
row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b))
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_split_shard_4(self):
'''Test the split of shard 4 and the global server configuration
after that. The test splits shard 4 between GROUPID5 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_db3_t3 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0])
row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0])
row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0])
status = self.proxy.sharding.split_shard("4", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
self.__server_5.connect()
row_cnt_shard_after_split_1_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_a = \
int(row_cnt_shard_after_split_1_db1_t1[0][0])
row_cnt_shard_after_split_1_db2_t2_a = \
int(row_cnt_shard_after_split_1_db2_t2[0][0])
row_cnt_shard_after_split_1_db3_t3_a = \
int(row_cnt_shard_after_split_1_db3_t3[0][0])
row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db3.t3", {"fetch": True})
row_cnt_shard_after_split_1_db1_t1_b = \
int(row_cnt_shard_after_split_1_db1_t1_b[0][0])
row_cnt_shard_after_split_1_db2_t2_b = \
int(row_cnt_shard_after_split_1_db2_t2_b[0][0])
row_cnt_shard_after_split_1_db3_t3_b = \
int(row_cnt_shard_after_split_1_db3_t3_b[0][0])
self.assertTrue(
row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a +
row_cnt_shard_after_split_1_db1_t1_b))
self.assertTrue(
row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a +
row_cnt_shard_after_split_1_db2_t2_b))
self.assertTrue(
row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a +
row_cnt_shard_after_split_1_db3_t3_b))
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_hash_dump(self):
"""Test the dump of the HASH sharding information for the table.
"""
shardinginformation_1 = tests.utils.make_info_result(
[[
'db1', 't1', 'userID', 'E2996A7B8509367020B55A4ACD2AE46A', '1',
'HASH', 'GROUPID2', 'GROUPID1'
],
[
'db1', 't1', 'userID', 'E88F547DF45C99F27646C76316FB21DF',
'2', 'HASH', 'GROUPID3', 'GROUPID1'
],
[
'db1', 't1', 'userID', '7A2E76448FF04233F3851A492BEF1090',
'3', 'HASH', 'GROUPID4', 'GROUPID1'
],
[
'db1', 't1', 'userID', '97427AA63E300F56536710F5D73A35FA',
'4', 'HASH', 'GROUPID5', 'GROUPID1'
]])
packet = self.proxy.dump.sharding_information(0, "db1.t1")
self.check_xmlrpc_result(packet, shardinginformation_1)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestShardingPrune(tests.utils.TestCase):
"""Contains unit tests for testing the shard move operation and for
verifying that the global server configuration remains constant after
the shard move configuration.
"""
def check_xmlrpc_command_success(self, packet, expect, has_error):
result = _xmlrpc._decode(packet)
self.assertEqual(bool(result.error), has_error,
"Had error '%s'" % result.error)
# Check that it returned the expected result (if one is
# expected), which is in the first result set (containing
# execution result), first row (there is just one for
# procedures), and last column (containing the result).
if expect is not None:
self.assertEqual(result.results[0][0][3], expect)
# If the call was successful, check that there is at least 2
# result sets and that the second result set contain more than
# zero jobs.
if not has_error:
self.assertTrue(len(result.results) > 1, str(result))
self.assertNotEqual(result.results[1].rowcount, 0,
"had %d result sets" % len(result.results))
def setUp(self):
"""Creates the topology for testing.
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__server_1.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF")
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF")
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71, 10):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301, 10):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201, 10):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201, 10):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201, 10):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71, 10):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301, 10):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201, 10):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201, 10):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201, 10):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF")
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71, 10):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301, 10):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201, 10):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201, 10):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201, 10):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71, 10):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301, 10):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201, 10):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201, 10):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201, 10):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(3),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF")
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71, 10):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301, 10):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201, 10):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201, 10):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201, 10):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71, 10):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301, 10):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201, 10):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201, 10):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201, 10):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add(self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(4),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF")
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71, 10):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301, 10):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201, 10):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201, 10):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201, 10):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71, 10):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(101, 301, 10):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(1001, 1201, 10):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(10001, 10201, 10):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
for i in range(100001, 100201, 10):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add(self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(5),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF")
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add(self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
packet = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.check_xmlrpc_command_success(packet, 1, False)
packet = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.check_xmlrpc_command_success(packet, None, False)
packet = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.check_xmlrpc_command_success(packet, None, False)
packet = self.proxy.sharding.add_shard(
1, "GROUPID2/1,GROUPID3/101,GROUPID4/1001,GROUPID5/10001",
"ENABLED")
self.check_xmlrpc_command_success(packet, None, False)
packet = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_success(packet, None, False)
def test_move_shard_1(self):
'''Test the move of shard 1 and the global server configuration
after that. The test moves shard 1 from GROUPID2 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID2 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_before_move_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_move_db1_t1, 7)
self.assertEqual(row_cnt_shard_before_move_db2_t2, 7)
packet = self.proxy.sharding.move_shard("1", "GROUPID6")
self.check_xmlrpc_command_success(packet, None, False)
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_after_move_db1_t1,
row_cnt_shard_before_move_db1_t1)
self.assertEqual(row_cnt_shard_after_move_db2_t2,
row_cnt_shard_before_move_db2_t2)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the first shard's old
#group.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertEqual(global_table_count, 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the first shard's new
#group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_2(self):
'''Test the move of shard 2 and the global server configuration
after that. The test moves shard 2 from GROUPID3 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID3 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_before_move_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_move_db1_t1, 20)
self.assertEqual(row_cnt_shard_before_move_db2_t2, 20)
packet = self.proxy.sharding.move_shard("2", "GROUPID6")
self.check_xmlrpc_command_success(packet, None, False)
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_after_move_db1_t1,
row_cnt_shard_before_move_db1_t1)
self.assertEqual(row_cnt_shard_after_move_db2_t2,
row_cnt_shard_before_move_db2_t2)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the second shard's
#old group.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertEqual(global_table_count, 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard's new
#group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_3(self):
'''Test the move of shard 3 and the global server configuration
after that. The test moves shard 3 from GROUPID4 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID4 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_before_move_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_move_db1_t1, 20)
self.assertEqual(row_cnt_shard_before_move_db2_t2, 20)
packet = self.proxy.sharding.move_shard("3", "GROUPID6")
self.check_xmlrpc_command_success(packet, None, False)
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_after_move_db1_t1,
row_cnt_shard_before_move_db1_t1)
self.assertEqual(row_cnt_shard_after_move_db2_t2,
row_cnt_shard_before_move_db2_t2)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the third shard's
#original group.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertEqual(global_table_count, 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard
#new group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_4(self):
'''Test the move of shard 4 and the global server configuration
after that. The test moves shard 4 from GROUPID5 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID5 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_before_move_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_move_db1_t1, 40)
self.assertEqual(row_cnt_shard_before_move_db2_t2, 40)
packet = self.proxy.sharding.move_shard("4", "GROUPID6")
self.check_xmlrpc_command_success(packet, None, False)
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2", {"fetch": True})
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertEqual(row_cnt_shard_after_move_db1_t1,
row_cnt_shard_before_move_db1_t1)
self.assertEqual(row_cnt_shard_after_move_db2_t2,
row_cnt_shard_before_move_db2_t2)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the fourth shard's
#original group.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertEqual(global_table_count, 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard's new
#group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table", {"fetch": True})
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestHashMoveGlobal(tests.utils.TestCase):
"""Contains unit tests for testing the shard move operation and for
verifying that the global server configuration remains constant after
the shard move configuration.
"""
def setUp(self):
"""Configure the existing environment
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(
1,
"GROUPID2,GROUPID3,GROUPID4,GROUPID5",
"ENABLED"
)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
def test_move_shard_1(self):
'''Test the move of shard 1 and the global server configuration
after that. The test moves shard 1 from GROUPID2 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID2 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("1", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_2(self):
'''Test the move of shard 2 and the global server configuration
after that. The test moves shard 2 from GROUPID3 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID3 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("2", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_3(self):
'''Test the move of shard 3 and the global server configuration
after that. The test moves shard 3 from GROUPID4 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID4 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("3", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the third shard's
#original group.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard
#new group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_4(self):
'''Test the move of shard 4 and the global server configuration
after that. The test moves shard 4 from GROUPID5 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID5 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("4", "GROUPID6")
self.check_xmlrpc_command_result(status)
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the fourth shard's
#original group.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard's new
#group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestMySQLSlave(unittest.TestCase):
"""Unit test for the configuration file handling.
"""
def setUp(self):
"""Configure the existing environment
"""
uuid = MySQLServer.discover_uuid(OPTIONS_MASTER["address"])
OPTIONS_MASTER["uuid"] = _uuid.UUID(uuid)
self.master = MySQLServer(**OPTIONS_MASTER)
self.master.connect()
reset_master(self.master)
uuid = MySQLServer.discover_uuid(OPTIONS_SLAVE["address"])
OPTIONS_SLAVE["uuid"] = _uuid.UUID(uuid)
self.slave = MySQLServer(**OPTIONS_SLAVE)
self.slave.connect()
stop_slave(self.slave, wait=True)
reset_master(self.slave)
reset_slave(self.slave, clean=True)
def tearDown(self):
"""Clean up the existing environment
"""
cleanup_environment()
stop_slave(self.slave, wait=True)
self.slave.disconnect()
self.master.disconnect()
def test_switch_master_(self):
"""Test the switch_master() function.
"""
# Note this is only being tested with gtids.
# Set up replication.
master = self.master
slave = self.slave
# Check that is slave is not connected to any master.
self.assertFalse(is_slave_thread_running(slave, (IO_THREAD, )))
self.assertNotEqual(slave_has_master(slave), str(master.uuid))
# Switch to a master.
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd)
start_slave(slave, wait=True)
self.assertTrue(is_slave_thread_running(slave, (IO_THREAD, )))
# The IO_THREAD status and the UUID are not atomically updated
# for that reason master_uuid can be None.
master_uuid = slave_has_master(slave)
self.assertTrue(
master_uuid == None or master_uuid == str(master.uuid)
)
# It is not possible to switch when replication is running.
self.assertRaises(_errors.DatabaseError, switch_master, slave,
master, MySQLInstances().user, MySQLInstances().passwd
)
# Reset and try to reconnect master and slave.
stop_slave(slave, wait=True)
reset_slave(slave, clean=True)
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd
)
start_slave(slave, wait=True)
self.assertTrue(is_slave_thread_running(slave, (IO_THREAD, )))
self.assertEqual(slave_has_master(slave), str(master.uuid))
# Change master's password, reset and try to reconnect master
# and slave.
stop_slave(slave, wait=True)
master.set_session_binlog(False)
master.exec_stmt(
"SET PASSWORD FOR '{user}'@'%%' = PASSWORD('foobar')".format(
user=MySQLInstances().user)
)
master.exec_stmt("FLUSH PRIVILEGES")
master.set_session_binlog(True)
switch_master(slave, master, MySQLInstances().user, "foobar")
start_slave(slave, wait=True)
self.assertTrue(is_slave_thread_running(slave, (IO_THREAD, )))
self.assertEqual(slave_has_master(slave), str(master.uuid))
# Reset master's password, reset and try to reconnect master
# and slave.
stop_slave(slave, wait=True)
master.set_session_binlog(False)
master.exec_stmt(
"SET PASSWORD FOR '{user}'@'%%' = "
"PASSWORD('{passwd}')".format(user=MySQLInstances().user,
passwd=MySQLInstances().passwd or "")
)
master.exec_stmt("FLUSH PRIVILEGES")
master.set_session_binlog(True)
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd
)
start_slave(slave, wait=True)
self.assertTrue(is_slave_thread_running(slave, (IO_THREAD, )))
self.assertEqual(slave_has_master(slave), str(master.uuid))
def test_start_stop(self):
"""Test start/stop slave functions.
"""
# Set up replication.
master = self.master
slave = self.slave
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd
)
# Start SQL Thread.
start_slave(slave, wait=True, threads=(SQL_THREAD, ))
status = get_slave_status(slave)
self.assertEqual(status[0].Slave_IO_Running.upper(), "NO")
self.assertEqual(status[0].Slave_SQL_Running.upper(), "YES")
# Start IO Thread.
start_slave(slave, wait=True, threads=(IO_THREAD, ))
status = get_slave_status(slave)
self.assertEqual(status[0].Slave_IO_Running.upper(), "YES")
self.assertEqual(status[0].Slave_SQL_Running.upper(), "YES")
# Stop IO Thread
stop_slave(slave, wait=True, threads=(IO_THREAD, ))
status = get_slave_status(slave)
self.assertEqual(status[0].Slave_IO_Running.upper(), "NO")
self.assertEqual(status[0].Slave_SQL_Running.upper(), "YES")
# Stop IO Thread
stop_slave(slave, wait=True, threads=(SQL_THREAD, ))
status = get_slave_status(slave)
self.assertEqual(status[0].Slave_IO_Running.upper(), "NO")
self.assertEqual(status[0].Slave_SQL_Running.upper(), "NO")
def test_wait_for_slave(self):
"""Test wait_for_slave_thread function.
"""
# Set up replication.
master = self.master
slave = self.slave
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd
)
# Wait for SQL Thread and IO Thread to stop. This times out.
start_slave(slave, wait=True)
self.assertRaises(_errors.TimeoutError, wait_for_slave_thread, slave,
timeout=1, wait_for_running=False)
# Wait for SQL Thread and IO Thread to start. This times out.
stop_slave(slave, wait=True)
self.assertRaises(_errors.TimeoutError, wait_for_slave_thread, slave,
timeout=1, wait_for_running=True)
master.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
master.exec_stmt("USE test")
master.exec_stmt("DROP TABLE IF EXISTS test")
master.exec_stmt("CREATE TABLE test(id INTEGER)")
binlog = get_master_status(master)
binlog_file = binlog[0][0]
binlog_pos = int(binlog[0][1])
# Wait until the slave catches up. It returns False because
# the threads are stopped.
self.assertRaises(_errors.DatabaseError, wait_for_slave, slave,
binlog_file, binlog_pos, timeout=0)
# Wait until the slave catches up. It returns True because
# the threads are running.
start_slave(slave, wait=True)
wait_for_slave(slave, binlog_file, binlog_pos, timeout=0)
# This times out because there are no new events.
self.assertRaises(_errors.TimeoutError, wait_for_slave, slave,
binlog_file, 2 * binlog_pos, timeout=3)
stop_slave(slave, wait=True)
master.exec_stmt("DROP TABLE IF EXISTS test")
master.exec_stmt("CREATE TABLE test(id INTEGER)")
# It throws an error because the threads are stopped.
self.assertRaises(_errors.DatabaseError, sync_slave_with_master,
slave, master, timeout=0)
# Wait until the slave catches up. It does not throw an error
# because the threads are running.
start_slave(slave, wait=True)
sync_slave_with_master(slave, master, timeout=0)
# This times out because there are no new events.
gtid_executed = "%s:%s" % (str(master.uuid), "1-20")
self.assertRaises(_errors.TimeoutError, wait_for_slave_gtid, slave,
gtid_executed, timeout=3)
def test_check_rpl_health(self):
"""Test check_slave_issues() function.
"""
# Set up replication.
master = self.master
slave = self.slave
slave.disconnect()
# Try to check the health when one cannot connect to the server.
ret = check_slave_issues(slave)
self.assertEqual(ret, {'is_running': False})
# Try to check the health when change master has not been executed.
slave.connect()
ret = check_slave_issues(slave)
self.assertEqual(ret, {'is_configured': False})
# Try to check the health after executing change master.
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd
)
ret = check_slave_issues(slave)
self.assertEqual(ret, {'io_running': False, 'sql_running': False})
# Try to check the health after starting one thread.
start_slave(slave, wait=True, threads=(SQL_THREAD, ))
ret = check_slave_issues(slave)
self.assertEqual(ret, {'io_running': False})
# Create data and synchronize to show there is no gtid behind.
master.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
master.exec_stmt("USE test")
master.exec_stmt("DROP TABLE IF EXISTS test")
master.exec_stmt("CREATE TABLE test(id INTEGER)")
start_slave(slave, wait=True, threads=(IO_THREAD, ))
sync_slave_with_master(slave, master, timeout=0)
ret = check_slave_delay(slave, master)
self.assertEqual(ret, {})
def test_get_gtid_behind(self):
"""Test get_gtid_behind() function.
"""
# Set up replication.
master = self.master
slave = self.slave
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd
)
# Check gtid that has no information on server_uuid.
self.assertRaises(_errors.ProgrammingError, get_num_gtid, "1")
sid_1 = "80139491-08ed-11e2-b7bd-f0def124dcc5"
sid_2 = "99939491-08ed-11e2-b7bd-f0def124dcc5"
# Check the pattern sid:trx_id.
ret = get_num_gtid("%s:%s" % (sid_1, "5"))
self.assertEqual(ret, 1)
# Check the pattern sid:trx_id-trx_id.
ret = get_num_gtid("%s:%s" % (sid_1, "5-10"))
self.assertEqual(ret, 6)
# Check the pattern sid:trx_id-trx_id, trx_id, trx_id-trx-id.
ret = get_num_gtid("%s:%s,%s,%s" % (sid_1, "5-10", "20", "25-30"))
self.assertEqual(ret, 13)
# Check the pattern sid:trx_id-trx_id, sid:trx_id-trx-id.
ret = get_num_gtid("%s:%s,%s:%s" % (sid_1, "5-10", sid_2, "5-6"))
self.assertEqual(ret, 8)
# Check the pattern sid:trx_id-trx_id, sid:trx_id-trx-id but filtering
# server_uuids that are different from sid_2.
ret = get_num_gtid("%s:%s,%s:%s" %
(sid_1, "5-10", sid_2, "5-6"), sid_2)
self.assertEqual(ret, 2)
# Check empty master_gtid_status and empty slave_gtid_status.
master_gtid_status = master.get_gtid_status()
slave_gtid_status = slave.get_gtid_status()
ret = get_slave_num_gtid_behind(slave, master_gtid_status)
self.assertEqual(ret, 0)
self.assertEqual(slave_gtid_status[0].GTID_EXECUTED, "")
self.assertEqual(master_gtid_status[0].GTID_EXECUTED, "")
# It is not possible to do any comparison if the master_gtid_status
# is empty.
slave.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
slave.exec_stmt("USE test")
slave.exec_stmt("DROP TABLE IF EXISTS test")
master_gtid_status = master.get_gtid_status()
slave_gtid_status = slave.get_gtid_status()
self.assertRaises(_errors.InvalidGtidError, get_slave_num_gtid_behind,
slave, master_gtid_status)
self.assertNotEqual(slave_gtid_status[0].GTID_EXECUTED, "")
self.assertEqual(master_gtid_status[0].GTID_EXECUTED, "")
# Check what happens if there are different sets of transactions.
master.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
master_gtid_status = master.get_gtid_status()
slave_gtid_status = slave.get_gtid_status()
ret = get_slave_num_gtid_behind(slave, master_gtid_status)
self.assertEqual(ret, 1)
self.assertNotEqual(slave_gtid_status[0].GTID_EXECUTED, "")
self.assertNotEqual(master_gtid_status[0].GTID_EXECUTED, "")
# Check what happens if the slave_gtid_status is empty.
reset_master(slave)
master_gtid_status = master.get_gtid_status()
slave_gtid_status = slave.get_gtid_status()
ret = get_slave_num_gtid_behind(slave, master_gtid_status)
self.assertEqual(ret, 1)
self.assertEqual(slave_gtid_status[0].GTID_EXECUTED, "")
self.assertNotEqual(master_gtid_status[0].GTID_EXECUTED, "")
class TestShardingPrune(tests.utils.TestCase):
def setUp(self):
"""Creates the following topology for testing,
GROUPID1 - localhost:13001, localhost:13002 - Global Group
GROUPID2 - localhost:13003, localhost:13004 - shard 1
GROUPID3 - localhost:13005, localhost:13006 - shard 2
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(3),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add(self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(4),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add(self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(5),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("CREATE DATABASE db1")
self.__server_6.exec_stmt("CREATE TABLE db1.t1"
"(userID INT, name VARCHAR(30))")
for i in range(1, 71):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(101, 301):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(1001, 1201):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(10001, 10201):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
for i in range(100001, 100201):
self.__server_6.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add(self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(
1, "GROUPID2/1,GROUPID3/101,GROUPID4/1001,"
"GROUPID5/10001,GROUPID6/100001", "ENABLED")
self.check_xmlrpc_command_result(status)
def test_prune_table_not_exists(self):
"""Delete the data in the database and verify that the prune does
not fail once the database has been removed.
"""
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1")
status = self.proxy.sharding.prune_shard("db1.t1")
def test_prune_shard(self):
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", 1, "LOCAL")
info = self.check_xmlrpc_simple(status, {}, rowcount=1)
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 70)
rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 70)
rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 1)
status = self.proxy.sharding.lookup_servers("db1.t1", 101, "LOCAL")
info = self.check_xmlrpc_simple(status, {}, rowcount=1)
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 300)
rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 101)
status = self.proxy.sharding.lookup_servers("db1.t1", 1202, "LOCAL")
info = self.check_xmlrpc_simple(status, {}, rowcount=1)
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 1200)
rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 1001)
status = self.proxy.sharding.lookup_servers("db1.t1", 11000, "LOCAL")
info = self.check_xmlrpc_simple(status, {}, rowcount=1)
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 10200)
rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 10001)
status = self.proxy.sharding.lookup_servers("db1.t1", 100100, "LOCAL")
info = self.check_xmlrpc_simple(status, {}, rowcount=1)
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 100200)
rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 100001)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestSharding(unittest.TestCase):
def setUp(self):
"""Configure the existing environment
"""
self.__options_1 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_1.mysql.com:3060",
}
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa75a12a-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_2.mysql.com:3060",
}
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
self.__group_1.add_server(self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS prune_db")
self.__server_3.exec_stmt("CREATE DATABASE prune_db")
self.__server_3.exec_stmt("CREATE TABLE prune_db.prune_table"
"(userID INT, name VARCHAR(30))")
self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(101, 'TEST 1')")
self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(202, 'TEST 2')")
self.__options_4 = {
"uuid" : _uuid.UUID("{dd75a12a-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_4.mysql.com:3060",
}
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_3)
self.__group_2.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_3)
self.__options_5 = {
"uuid" : _uuid.UUID("{ee75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS prune_db")
self.__server_5.exec_stmt("CREATE DATABASE prune_db")
self.__server_5.exec_stmt("CREATE TABLE prune_db.prune_table"
"(userID INT, name VARCHAR(30))")
self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(101, 'TEST 1')")
self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table "
"VALUES(202, 'TEST 2')")
self.__options_6 = {
"uuid" : _uuid.UUID("{ff75a12a-98d1-414c-96af-9e9d4b179678}"),
"address" : "server_6.mysql.com:3060",
}
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_5)
self.__group_3.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_5)
group_4 = Group("GROUPID4", "4TH description.")
Group.add(group_4)
group_5 = Group("GROUPID5", "5TH description.")
Group.add(group_5)
group_6 = Group("GROUPID6", "6TH description.")
Group.add(group_6)
group_7 = Group("GROUPID7", "7TH description.")
Group.add(group_7)
group_8 = Group("GROUPID8", "8TH description.")
Group.add(group_8)
group_9 = Group("GROUPID9", "9TH description.")
Group.add(group_9)
group_10 = Group("GROUPID10", "10TH description.")
Group.add(group_10)
group_11 = Group("GROUPID11", "11TH description.")
Group.add(group_11)
group_12 = Group("GROUPID12", "12TH description.")
Group.add(group_12)
group_13 = Group("GROUPID13", "13TH description.")
Group.add(group_13)
group_14 = Group("GROUPID14", "14TH description.")
Group.add(group_14)
self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEquals( self.__shard_mapping_list, [])
self.__shard_mapping_id_1 = ShardMapping.define("RANGE", "GROUPID10")
self.__shard_mapping_id_2 = ShardMapping.define("RANGE", "GROUPID11")
self.__shard_mapping_id_3 = ShardMapping.define("RANGE", "GROUPID12")
#Test with sharding type values in lower case
self.__shard_mapping_id_4 = ShardMapping.define("range", "GROUPID13")
self.__shard_mapping_id_5 = ShardMapping.define("range", "GROUPID14")
self.__shard_mapping_1 = \
ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID1")
self.__shard_mapping_2 = \
ShardMapping.add(self.__shard_mapping_id_2, "db2.t2", "userID2")
self.__shard_mapping_3 = \
ShardMapping.add(self.__shard_mapping_id_3, "db3.t3", "userID3")
self.__shard_mapping_4 = \
ShardMapping.add(self.__shard_mapping_id_4, "db4.t4", "userID4")
self.__shard_mapping_5 = \
ShardMapping.add(self.__shard_mapping_id_5, "prune_db.prune_table",
"userID")
self.__shard_id_1 = Shards.add("GROUPID1")
self.__shard_id_2 = Shards.add("GROUPID10")
self.__shard_id_3 = Shards.add("GROUPID11")
self.__shard_id_4 = Shards.add("GROUPID4")
self.__shard_id_5 = Shards.add("GROUPID5")
self.__shard_id_6 = Shards.add("GROUPID6")
self.__shard_id_7 = Shards.add("GROUPID7")
self.__shard_id_8 = Shards.add("GROUPID8")
self.__shard_id_9 = Shards.add("GROUPID9")
self.__shard_id_10 = Shards.add("GROUPID2")
self.__shard_id_11 = Shards.add("GROUPID3")
self.__range_sharding_specification_1 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
0,
self.__shard_id_1.shard_id
)
self.__range_sharding_specification_2 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
1001,
self.__shard_id_2.shard_id
)
self.__range_sharding_specification_3 = RangeShardingSpecification.add(
self.__shard_mapping_1.shard_mapping_id,
2001,
self.__shard_id_3.shard_id
)
self.__range_sharding_specification_4 = RangeShardingSpecification.add(
self.__shard_mapping_2.shard_mapping_id,
3001,
self.__shard_id_4.shard_id
)
self.__range_sharding_specification_5 = RangeShardingSpecification.add(
self.__shard_mapping_2.shard_mapping_id,
4001,
self.__shard_id_5.shard_id
)
self.__range_sharding_specification_6 = RangeShardingSpecification.add(
self.__shard_mapping_3.shard_mapping_id,
6001,
self.__shard_id_6.shard_id
)
self.__range_sharding_specification_7 = RangeShardingSpecification.add(
self.__shard_mapping_3.shard_mapping_id,
7001,
self.__shard_id_7.shard_id
)
self.__range_sharding_specification_8 = RangeShardingSpecification.add(
self.__shard_mapping_4.shard_mapping_id,
8001,
self.__shard_id_8.shard_id
)
self.__range_sharding_specification_9 = RangeShardingSpecification.add(
self.__shard_mapping_4.shard_mapping_id,
10001,
self.__shard_id_9.shard_id
)
self.__range_sharding_specification_10 = RangeShardingSpecification.add(
self.__shard_mapping_5.shard_mapping_id,
100,
self.__shard_id_10.shard_id)
self.__range_sharding_specification_11 = RangeShardingSpecification.add(
self.__shard_mapping_5.shard_mapping_id,
201,
self.__shard_id_11.shard_id)
def tearDown(self):
"""Clean up the existing environment
"""
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS prune_db")
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS prune_db")
tests.utils.cleanup_environment()
def test_fetch_shard_mapping(self):
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_2 = ShardMapping.fetch("db2.t2")
shard_mapping_3 = ShardMapping.fetch("db3.t3")
shard_mapping_4 = ShardMapping.fetch("db4.t4")
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_1, shard_mapping_1))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_2, shard_mapping_2))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_3, shard_mapping_3))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_4, shard_mapping_4))
def test_fetch_sharding_scheme(self):
range_sharding_specifications = RangeShardingSpecification.list(1)
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[0],
self.__range_sharding_specification_1))
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[1],
self.__range_sharding_specification_2))
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[2],
self.__range_sharding_specification_3))
def test_lookup_sharding_scheme(self):
r_spec_1 = \
RangeShardingSpecification.lookup(500, self.__shard_mapping_id_1,
"RANGE")
self.assertEqual(r_spec_1.shard_id, self.__shard_id_1.shard_id)
r_spec_2 = \
RangeShardingSpecification.lookup(3500, self.__shard_mapping_id_2,
"RANGE")
self.assertEqual(r_spec_2.shard_id, self.__shard_id_4.shard_id)
r_spec_3 = \
RangeShardingSpecification.lookup(6500, self.__shard_mapping_id_3,
"RANGE")
self.assertEqual(r_spec_3.shard_id, self.__shard_id_6.shard_id)
def test_shard_mapping_list_mappings(self):
shard_mappings = ShardMapping.list("RANGE")
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_1, shard_mappings[0]))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_2, shard_mappings[1]))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_3, shard_mappings[2]))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_4, shard_mappings[3]))
def test_shard_mapping_getters(self):
self.assertEqual(self.__shard_mapping_1.table_name, "db1.t1")
self.assertEqual(self.__shard_mapping_1.column_name, "userID1")
self.assertEqual(self.__shard_mapping_1.type_name, "RANGE")
self.assertEqual(self.__shard_mapping_1.shard_mapping_id, 1)
self.assertEqual(self.__shard_mapping_1.global_group, "GROUPID10")
def test_shard_mapping_remove(self):
self.__range_sharding_specification_1.remove()
self.__range_sharding_specification_2.remove()
self.__range_sharding_specification_3.remove()
self.__range_sharding_specification_4.remove()
self.__range_sharding_specification_5.remove()
self.__range_sharding_specification_6.remove()
self.__range_sharding_specification_7.remove()
self.__range_sharding_specification_8.remove()
self.__range_sharding_specification_9.remove()
self.__range_sharding_specification_10.remove()
self.__shard_id_1.remove()
self.__shard_id_2.remove()
self.__shard_id_3.remove()
self.__shard_id_4.remove()
self.__shard_id_5.remove()
self.__shard_id_6.remove()
self.__shard_id_7.remove()
self.__shard_id_8.remove()
self.__shard_id_9.remove()
self.__shard_id_10.remove()
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_1.remove()
def test_range_sharding_specification_getters(self):
self.assertEqual(
self.__range_sharding_specification_1.shard_mapping_id, 1)
self.assertEqual(self.__range_sharding_specification_1.lower_bound, 0)
self.assertEqual(self.__range_sharding_specification_1.shard_id, 1)
def test_list_shard_mapping(self):
expected_shard_mapping_list1 = [1, "RANGE", "GROUPID10"]
expected_shard_mapping_list2 = [2, "RANGE", "GROUPID11"]
expected_shard_mapping_list3 = [3, "RANGE", "GROUPID12"]
expected_shard_mapping_list4 = [4, "RANGE", "GROUPID13"]
expected_shard_mapping_list5 = [5, "RANGE", "GROUPID14"]
obtained_shard_mapping_list = ShardMapping.list_shard_mapping_defn()
self.assertEqual(
set(expected_shard_mapping_list1),
set(obtained_shard_mapping_list[0])
)
self.assertEqual(
set(expected_shard_mapping_list2),
set(obtained_shard_mapping_list[1])
)
self.assertEqual(
set(expected_shard_mapping_list3),
set(obtained_shard_mapping_list[2])
)
self.assertEqual(
set(expected_shard_mapping_list4),
set(obtained_shard_mapping_list[3])
)
self.assertEqual(
set(expected_shard_mapping_list5),
set(obtained_shard_mapping_list[4])
)
class TestShardingPrune(tests.utils.TestCase):
"""Contains unit tests for testing the shard split operation and for
verifying that the global server configuration remains constant after
the shard split configuration.
"""
def setUp(self):
"""Creates the topology for testing.
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20) PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES('%s', 'TEST %s')" % ("e"+str(i), i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID VARCHAR(20), salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 71):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("a"+str(i), i))
for i in range(101, 301):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("b"+str(i), i))
for i in range(1001, 1201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("c"+str(i), i))
for i in range(10001, 10201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("d"+str(i), i))
for i in range(100001, 100201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES('%s', %s)" % ("e"+str(i), i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__server_6.connect()
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("RANGE_STRING", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(
1,
"GROUPID2/a,GROUPID3/b,GROUPID4/c,GROUPID5/d",
"ENABLED"
)
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.prune_shard("db1.t1")
self.check_xmlrpc_command_result(status)
def test_split_shard_4(self):
'''Test the split of shard 4 and the global server configuration
after that. The test splits shard 4 between GROUPID5 and GROUPID6.
After the split is done, it verifies the count on GROUPID6 to check
that the group has tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples.
'''
row_cnt_shard_before_split_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_split_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_split_db1_t1 = \
int(row_cnt_shard_before_split_db1_t1[0][0])
row_cnt_shard_before_split_db2_t2 = \
int(row_cnt_shard_before_split_db2_t2[0][0])
self.assertEqual(row_cnt_shard_before_split_db1_t1, 400)
self.assertEqual(row_cnt_shard_before_split_db2_t2, 400)
status = self.proxy.sharding.split_shard("4", "GROUPID6", "e")
self.check_xmlrpc_command_result(status)
self.__server_5.connect()
row_cnt_shard_after_split_db1_t1_a = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_a = \
int(row_cnt_shard_after_split_db1_t1_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 200)
row_cnt_shard_after_split_db2_t2_a = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_a = \
int(row_cnt_shard_after_split_db2_t2_a[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 200)
row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_split_db1_t1_b = \
int(row_cnt_shard_after_split_db1_t1_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 200)
row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_split_db2_t2_b = \
int(row_cnt_shard_after_split_db2_t2_b[0][0])
self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 200)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID VARCHAR(20), name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES('%s', 'TEST %s')" % (i, i))
time.sleep(5)
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestShardingServices(tests.utils.TestCase):
def setUp(self):
"""Configure the existing environment
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(0),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(1),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(2),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add(self.__server_3)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add(self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(3),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add(self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__options_5 = {
"uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(4),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add(self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__options_6 = {
"uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address": MySQLInstances().get_address(5),
"user": MySQLInstances().user,
"passwd": MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add(self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=1)
status = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=2)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.check_xmlrpc_command_result(status, returns=3)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID1")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(2, "db2.t2", "userID2")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_table(3, "db3.t3", "userID3")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(1, "GROUPID2,GROUPID3",
"ENABLED")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(2, "GROUPID4/0,GROUPID5/101",
"ENABLED")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.add_shard(3, "GROUPID6", "ENABLED")
self.check_xmlrpc_command_result(status)
def test_list_shard_defns(self):
expected_shard_mapping_list1 = tests.utils.make_mapping_result(
[[1, "HASH", "GROUPID1"], [2, "RANGE", "GROUPID1"],
[3, "HASH", "GROUPID1"]])
status = self.proxy.sharding.list_definitions()
self.check_xmlrpc_result(status, expected_shard_mapping_list1)
def test_list_shard_mappings(self):
expected_hash = tests.utils.make_shard_mapping_list_result([
[1, 'HASH', "db1.t1", "GROUPID1", "userID1"],
[3, 'HASH', "db3.t3", "GROUPID1", "userID3"],
])
status = self.proxy.sharding.list_tables("HASH")
self.check_xmlrpc_result(status, expected_hash)
expected_range = tests.utils.make_shard_mapping_list_result([
[2, "RANGE", "db2.t2", "GROUPID1", "userID2"],
])
status = self.proxy.sharding.list_tables("RANGE")
self.check_xmlrpc_result(status, expected_range)
def test_shard_prune(self):
status = self.proxy.sharding.prune_shard("db2.t2")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db2.t2", 1, "LOCAL")
info = self.check_xmlrpc_simple(status, {})
shard_uuid = info['server_uuid']
shard_server = fetch_test_server(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db2.t2",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt("SELECT MAX(userID2) FROM db2.t2",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt("SELECT MIN(userID2) FROM db2.t2",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 1)
status = self.proxy.sharding.lookup_servers("db2.t2", 101, "LOCAL")
info = self.check_xmlrpc_simple(status, {})
shard_uuid = info['server_uuid']
shard_server = fetch_test_server(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db2.t2",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt("SELECT MAX(userID2) FROM db2.t2",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt("SELECT MIN(userID2) FROM db2.t2",
{"fetch": True})
self.assertTrue(int(rows[0][0]) == 101)
def tearDown(self):
"""Clean up the existing environment
"""
tests.utils.cleanup_environment()
class TestShardingServices(unittest.TestCase):
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Configure the existing environment
"""
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID2 INT, name VARCHAR(30))")
for i in range(1, 201):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.create_definition("RANGE", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 2)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 3)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(2, "db2.t2", "userID2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(3, "db3.t3", "userID3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(1, "GROUPID2,GROUPID3",
"ENABLED")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
status = self.proxy.sharding.add_shard(2, "GROUPID4/0,GROUPID5/101",
"ENABLED")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
status = self.proxy.sharding.add_shard(3, "GROUPID6", "ENABLED")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
def test_list_shard_defns(self):
expected_shard_mapping_list1 = [
[1, "HASH", "GROUPID1"],
[2, "RANGE", "GROUPID1"],
[3, "HASH", "GROUPID1"]
]
status = self.proxy.sharding.list_definitions()
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_shard_mapping_list = status[2]
self.assertEqual(set(expected_shard_mapping_list1[0]),
set(obtained_shard_mapping_list[0]))
self.assertEqual(set(expected_shard_mapping_list1[1]),
set(obtained_shard_mapping_list[1]))
self.assertEqual(set(expected_shard_mapping_list1[2]),
set(obtained_shard_mapping_list[2]))
def test_list_shard_mappings(self):
status = self.proxy.sharding.list_tables("HASH")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], [
{"shard_mapping_id":1,
"table_name":"db1.t1",
"column_name":"userID1",
"type_name":"HASH",
"global_group":"GROUPID1"},
{"shard_mapping_id":3,
"table_name":"db3.t3",
"column_name":"userID3",
"type_name":"HASH",
"global_group":"GROUPID1"}
])
status = self.proxy.sharding.list_tables("RANGE")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
self.assertEqual(status[2], [
{"shard_mapping_id":2,
"table_name":"db2.t2",
"column_name":"userID2",
"type_name":"RANGE",
"global_group":"GROUPID1"}
])
def test_shard_prune(self):
status = self.proxy.sharding.prune_shard("db2.t2")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
status = self.proxy.sharding.lookup_servers("db2.t2", 1, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MAX(userID2) FROM db2.t2",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MIN(userID2) FROM db2.t2",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 1)
status = self.proxy.sharding.lookup_servers("db2.t2", 101, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
shard_uuid = obtained_server_list[0][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 100)
rows = shard_server.exec_stmt(
"SELECT MAX(userID2) FROM db2.t2",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 200)
rows = shard_server.exec_stmt(
"SELECT MIN(userID2) FROM db2.t2",
{"fetch" : True})
self.assertTrue(int(rows[0][0]) == 101)
def tearDown(self):
"""Clean up the existing environment
"""
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.remove_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.remove_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.remove_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.remove_shard(4)
self.proxy.sharding.disable_shard(5)
self.proxy.sharding.remove_shard(5)
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
def test_privileges(self):
"""Test whether user's have the appropriate privileges.
"""
# Some privileges
MINIMUM_PRIVILEGES = [
"REPLICATION SLAVE", "REPLICATION CLIENT", "SUPER",
"SHOW DATABASES", "RELOAD"
]
# Connect to server as admin and create temporary user.
uuid = MySQLServer.discover_uuid(OPTIONS["address"])
server = MySQLServer(_uuid.UUID(uuid), OPTIONS["address"],
tests.utils.MySQLInstances().user,
tests.utils.MySQLInstances().passwd)
ConnectionManager().purge_connections(server)
server.connect()
server.set_session_binlog(False)
server.exec_stmt("CREATE USER 'jeffrey'@'%%' IDENTIFIED BY 'mypass'")
# Check if jeffrey (temporary user) has the appropriate privileges.
# There is not privilege associate to jeffrey.
new_server = MySQLServer(_uuid.UUID(uuid), OPTIONS["address"],
"jeffrey", "mypass")
new_server.connect()
self.assertFalse(new_server.has_privileges(MINIMUM_PRIVILEGES))
# Check if jeffrey (temporary user) has the appropriate privileges.
# Grant required privileges except RELOAD
# There is no RELOAD on a global level.
privileges = ", ".join(
[priv for priv in MINIMUM_PRIVILEGES if priv != "RELOAD"])
server.exec_stmt("GRANT {privileges} ON *.* TO 'jeffrey'@'%%'".format(
privileges=privileges))
self.assertFalse(new_server.has_privileges(MINIMUM_PRIVILEGES))
# Check if jeffrey (temporary user) has the appropriate privileges.
# The RELOAD on a global level was granted.
server.exec_stmt("GRANT RELOAD ON *.* TO 'jeffrey'@'%%'")
self.assertTrue(new_server.has_privileges(MINIMUM_PRIVILEGES))
# Check if jeffrey (temporary user) has the appropriate privileges.
# Revoke privilegs from temporary user.
# There is no ALL on a global level.
server.exec_stmt("REVOKE ALL PRIVILEGES, GRANT OPTION FROM "
"'jeffrey'@'%%'")
server.exec_stmt("GRANT ALL ON fabric.* TO 'jeffrey'@'%%'")
self.assertFalse(new_server.has_privileges(MINIMUM_PRIVILEGES))
# Check if jeffrey (temporary user) has the appropriate privileges.
# Grant all privileges, which the administrative user has to have.
server.exec_stmt("GRANT"
" ALTER, ALTER ROUTINE, CREATE, CREATE ROUTINE,"
" CREATE TEMPORARY TABLES, CREATE USER,"
" CREATE VIEW, DELETE, DROP, EVENT, EXECUTE,"
" GRANT OPTION, INDEX, INSERT, LOCK TABLES, PROCESS, "
" RELOAD, REPLICATION CLIENT, REPLICATION SLAVE,"
" SELECT, SHOW DATABASES, SHOW VIEW, SHUTDOWN,"
" SUPER, TRIGGER, UPDATE"
" ON *.* TO 'jeffrey'@'%%'")
self.assertTrue(new_server.has_privileges(MINIMUM_PRIVILEGES))
# Drop temporary user.
server.exec_stmt("DROP USER 'jeffrey'@'%%'")
server.set_session_binlog(True)
server.disconnect()
new_server.disconnect()
ConnectionManager().purge_connections(server)
class TestHashMoveGlobal(unittest.TestCase):
"""Contains unit tests for testing the shard move operation and for
verifying that the global server configuration remains constant after
the shard move configuration.
"""
def assertStatus(self, status, expect):
items = (item['diagnosis'] for item in status[1] if item['diagnosis'])
self.assertEqual(status[1][-1]["success"], expect, "\n".join(items))
def setUp(self):
"""Configure the existing environment
"""
tests.utils.cleanup_environment()
self.manager, self.proxy = tests.utils.setup_xmlrpc()
self.__options_1 = {
"uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(0),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"])
self.__options_1["uuid"] = _uuid.UUID(uuid_server1)
self.__server_1 = MySQLServer(**self.__options_1)
MySQLServer.add(self.__server_1)
self.__server_1.connect()
self.__group_1 = Group("GROUPID1", "First description.")
Group.add(self.__group_1)
self.__group_1.add_server(self.__server_1)
tests.utils.configure_decoupled_master(self.__group_1, self.__server_1)
self.__options_2 = {
"uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(1),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"])
self.__options_2["uuid"] = _uuid.UUID(uuid_server2)
self.__server_2 = MySQLServer(**self.__options_2)
MySQLServer.add(self.__server_2)
self.__server_2.connect()
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_2.exec_stmt("CREATE DATABASE db1")
self.__server_2.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_2.exec_stmt("CREATE DATABASE db2")
self.__server_2.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_2.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_2 = Group("GROUPID2", "Second description.")
Group.add(self.__group_2)
self.__group_2.add_server(self.__server_2)
tests.utils.configure_decoupled_master(self.__group_2, self.__server_2)
self.__options_3 = {
"uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(2),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
self.__server_3 = MySQLServer(**self.__options_3)
MySQLServer.add( self.__server_3)
self.__server_3.connect()
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_3.exec_stmt("CREATE DATABASE db1")
self.__server_3.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_3.exec_stmt("CREATE DATABASE db2")
self.__server_3.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_3.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_3 = Group("GROUPID3", "Third description.")
Group.add( self.__group_3)
self.__group_3.add_server(self.__server_3)
tests.utils.configure_decoupled_master(self.__group_3, self.__server_3)
self.__options_4 = {
"uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(3),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"])
self.__options_4["uuid"] = _uuid.UUID(uuid_server4)
self.__server_4 = MySQLServer(**self.__options_4)
MySQLServer.add(self.__server_4)
self.__server_4.connect()
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_4.exec_stmt("CREATE DATABASE db1")
self.__server_4.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_4.exec_stmt("CREATE DATABASE db2")
self.__server_4.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_4.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_4 = Group("GROUPID4", "Fourth description.")
Group.add( self.__group_4)
self.__group_4.add_server(self.__server_4)
tests.utils.configure_decoupled_master(self.__group_4, self.__server_4)
self.__options_5 = {
"uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(4),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
self.__server_5 = MySQLServer(**self.__options_5)
MySQLServer.add(self.__server_5)
self.__server_5.connect()
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1")
self.__server_5.exec_stmt("CREATE DATABASE db1")
self.__server_5.exec_stmt("CREATE TABLE db1.t1"
"(userID INT PRIMARY KEY, name VARCHAR(30))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db1.t1 "
"VALUES(%s, 'TEST %s')" % (i, i))
self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2")
self.__server_5.exec_stmt("CREATE DATABASE db2")
self.__server_5.exec_stmt("CREATE TABLE db2.t2"
"(userID INT, salary INT, "
"CONSTRAINT FOREIGN KEY(userID) "
"REFERENCES db1.t1(userID))")
for i in range(1, 1001):
self.__server_5.exec_stmt("INSERT INTO db2.t2 "
"VALUES(%s, %s)" % (i, i))
self.__group_5 = Group("GROUPID5", "Fifth description.")
Group.add( self.__group_5)
self.__group_5.add_server(self.__server_5)
tests.utils.configure_decoupled_master(self.__group_5, self.__server_5)
self.__options_6 = {
"uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"),
"address" : MySQLInstances().get_address(5),
"user" : MySQLInstances().user,
"passwd" : MySQLInstances().passwd,
}
uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"])
self.__options_6["uuid"] = _uuid.UUID(uuid_server6)
self.__server_6 = MySQLServer(**self.__options_6)
MySQLServer.add(self.__server_6)
self.__group_6 = Group("GROUPID6", "Sixth description.")
Group.add( self.__group_6)
self.__group_6.add_server(self.__server_6)
tests.utils.configure_decoupled_master(self.__group_6, self.__server_6)
status = self.proxy.sharding.create_definition("HASH", "GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_define_shard_mapping).")
self.assertEqual(status[2], 1)
status = self.proxy.sharding.add_table(1, "db1.t1", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_table(1, "db2.t2", "userID")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard_mapping).")
status = self.proxy.sharding.add_shard(
1,
"GROUPID2,GROUPID3,GROUPID4,GROUPID5",
"ENABLED"
)
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_add_shard).")
status = self.proxy.sharding.prune_shard("db1.t1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables).")
def test_move_shard_1(self):
'''Test the move of shard 1 and the global server configuration
after that. The test moves shard 1 from GROUPID2 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID2 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("1", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_setup_resharding_switch).")
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_2(self):
'''Test the move of shard 2 and the global server configuration
after that. The test moves shard 2 from GROUPID3 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID3 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("2", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_setup_resharding_switch).")
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fifth shard.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_3(self):
'''Test the move of shard 3 and the global server configuration
after that. The test moves shard 3 from GROUPID4 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID4 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("3", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_setup_resharding_switch).")
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the third shard's
#original group.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard
#new group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def test_move_shard_4(self):
'''Test the move of shard 4 and the global server configuration
after that. The test moves shard 4 from GROUPID5 to GROUPID6.
After the move is done, it verifies the count on GROUPID6 to check
that the group has all the tuples from the earlier group. Now it fires
an INSERT on the global group and verifies that all the shards have got
the inserted tuples. GROUPID5 should not have received the values
since it has had the shard moved away from it.
'''
row_cnt_shard_before_move_db1_t1 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_before_move_db2_t2 = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_before_move_db1_t1 = \
int(row_cnt_shard_before_move_db1_t1[0][0])
row_cnt_shard_before_move_db2_t2 = \
int(row_cnt_shard_before_move_db2_t2[0][0])
status = self.proxy.sharding.move_shard("4", "GROUPID6")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_setup_resharding_switch).")
self.__server_6.connect()
row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM db2.t2",
{"fetch" : True}
)
row_cnt_shard_after_move_db1_t1 = \
int(row_cnt_shard_after_move_db1_t1[0][0])
row_cnt_shard_after_move_db2_t2 = \
int(row_cnt_shard_after_move_db2_t2[0][0])
self.assertTrue(
row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1
)
self.assertTrue(
row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2
)
#Enter data into the global server
self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global")
self.__server_1.exec_stmt("CREATE DATABASE global")
self.__server_1.exec_stmt("CREATE TABLE global.global_table"
"(userID INT, name VARCHAR(30))")
for i in range(1, 11):
self.__server_1.exec_stmt("INSERT INTO global.global_table "
"VALUES(%s, 'TEST %s')" % (i, i))
time.sleep(5)
try:
#Verify that the data is not there in the fourth shard's
#original group.
global_table_count = self.__server_5.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
raise Exception("Server should not be connected to global server")
except _errors.DatabaseError:
pass
#Verify that the data is there in the first shard.
global_table_count = self.__server_2.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the second shard.
global_table_count = self.__server_3.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the third shard.
global_table_count = self.__server_4.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
#Verify that the data is there in the fourth shard's new
#group.
global_table_count = self.__server_6.exec_stmt(
"SELECT COUNT(*) FROM global.global_table",
{"fetch" : True}
)
global_table_count = int(global_table_count[0][0])
self.assertTrue(global_table_count == 10)
def tearDown(self):
"""Clean up the existing environment
"""
self.proxy.sharding.disable_shard(1)
self.proxy.sharding.remove_shard(1)
self.proxy.sharding.disable_shard(2)
self.proxy.sharding.remove_shard(2)
self.proxy.sharding.disable_shard(3)
self.proxy.sharding.remove_shard(3)
self.proxy.sharding.disable_shard(4)
self.proxy.sharding.remove_shard(4)
self.proxy.sharding.disable_shard(5)
self.proxy.sharding.remove_shard(5)
tests.utils.cleanup_environment()
tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestMySQLSlave(tests.utils.TestCase):
"""Unit test for the configuration file handling.
"""
def setUp(self):
"""Configure the existing environment
"""
uuid = MySQLServer.discover_uuid(OPTIONS_MASTER["address"])
OPTIONS_MASTER["uuid"] = _uuid.UUID(uuid)
self.master = MySQLServer(**OPTIONS_MASTER)
self.master.connect()
reset_master(self.master)
uuid = MySQLServer.discover_uuid(OPTIONS_SLAVE["address"])
OPTIONS_SLAVE["uuid"] = _uuid.UUID(uuid)
self.slave = MySQLServer(**OPTIONS_SLAVE)
self.slave.connect()
stop_slave(self.slave, wait=True)
reset_master(self.slave)
reset_slave(self.slave, clean=True)
def tearDown(self):
"""Clean up the existing environment
"""
cleanup_environment()
stop_slave(self.slave, wait=True)
self.slave.disconnect()
self.master.disconnect()
def test_switch_master_(self):
"""Test the switch_master() function.
"""
# Note this is only being tested with gtids.
# Set up replication.
master = self.master
slave = self.slave
# Check that is slave is not connected to any master.
self.assertFalse(is_slave_thread_running(slave, (IO_THREAD, )))
self.assertNotEqual(slave_has_master(slave), str(master.uuid))
# Switch to a master.
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd)
start_slave(slave, wait=True)
self.assertTrue(is_slave_thread_running(slave, (IO_THREAD, )))
# The IO_THREAD status and the UUID are not atomically updated
# for that reason master_uuid can be None.
master_uuid = slave_has_master(slave)
self.assertTrue(
master_uuid == None or master_uuid == str(master.uuid)
)
# It is not possible to switch when replication is running.
self.assertRaises(_errors.DatabaseError, switch_master, slave,
master, MySQLInstances().user, MySQLInstances().passwd
)
# Reset and try to reconnect master and slave.
stop_slave(slave, wait=True)
reset_slave(slave, clean=True)
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd
)
start_slave(slave, wait=True)
self.assertTrue(is_slave_thread_running(slave, (IO_THREAD, )))
self.assertEqual(slave_has_master(slave), str(master.uuid))
# Change master's password, reset and try to reconnect master
# and slave.
stop_slave(slave, wait=True)
master.set_session_binlog(False)
master.exec_stmt(
"SET PASSWORD = PASSWORD('foobar')"
)
master.set_session_binlog(True)
switch_master(slave, master, MySQLInstances().user, "foobar")
start_slave(slave, wait=True)
self.assertTrue(is_slave_thread_running(slave, (IO_THREAD, )))
self.assertEqual(slave_has_master(slave), str(master.uuid))
# Reset master's password, reset and try to reconnect master
# and slave.
stop_slave(slave, wait=True)
master.set_session_binlog(False)
master.exec_stmt(
"SET PASSWORD = PASSWORD('{passwd}')".format(
passwd=MySQLInstances().passwd or "")
)
master.set_session_binlog(True)
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd
)
start_slave(slave, wait=True)
self.assertTrue(is_slave_thread_running(slave, (IO_THREAD, )))
self.assertEqual(slave_has_master(slave), str(master.uuid))
def test_start_stop(self):
"""Test start/stop slave functions.
"""
# Set up replication.
master = self.master
slave = self.slave
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd
)
# Start SQL Thread.
start_slave(slave, wait=True, threads=(SQL_THREAD, ))
status = get_slave_status(slave)
self.assertEqual(status[0].Slave_IO_Running.upper(), "NO")
self.assertEqual(status[0].Slave_SQL_Running.upper(), "YES")
# Start IO Thread.
start_slave(slave, wait=True, threads=(IO_THREAD, ))
status = get_slave_status(slave)
self.assertEqual(status[0].Slave_IO_Running.upper(), "YES")
self.assertEqual(status[0].Slave_SQL_Running.upper(), "YES")
# Stop IO Thread
stop_slave(slave, wait=True, threads=(IO_THREAD, ))
status = get_slave_status(slave)
self.assertEqual(status[0].Slave_IO_Running.upper(), "NO")
self.assertEqual(status[0].Slave_SQL_Running.upper(), "YES")
# Stop IO Thread
stop_slave(slave, wait=True, threads=(SQL_THREAD, ))
status = get_slave_status(slave)
self.assertEqual(status[0].Slave_IO_Running.upper(), "NO")
self.assertEqual(status[0].Slave_SQL_Running.upper(), "NO")
def test_wait_for_slave(self):
"""Test wait_for_slave_thread function.
"""
# Set up replication.
master = self.master
slave = self.slave
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd
)
# Wait for SQL Thread and IO Thread to stop. This times out.
start_slave(slave, wait=True)
self.assertRaises(_errors.TimeoutError, wait_for_slave_thread, slave,
timeout=1, wait_for_running=False)
# Wait for SQL Thread and IO Thread to start. This times out.
stop_slave(slave, wait=True)
self.assertRaises(_errors.TimeoutError, wait_for_slave_thread, slave,
timeout=1, wait_for_running=True)
master.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
master.exec_stmt("USE test")
master.exec_stmt("DROP TABLE IF EXISTS test")
master.exec_stmt("CREATE TABLE test(id INTEGER)")
binlog = get_master_status(master)
binlog_file = binlog[0][0]
binlog_pos = binlog[0][1]
# Wait until the slave catches up. It returns False because
# the threads are stopped.
self.assertRaises(_errors.DatabaseError, wait_for_slave, slave,
binlog_file, binlog_pos, timeout=0)
# Wait until the slave catches up. It returns True because
# the threads are running.
start_slave(slave, wait=True)
wait_for_slave(slave, binlog_file, binlog_pos, timeout=0)
# This times out because there are no new events.
self.assertRaises(_errors.TimeoutError, wait_for_slave, slave,
binlog_file, 2 * binlog_pos, timeout=3)
stop_slave(slave, wait=True)
master.exec_stmt("DROP TABLE IF EXISTS test")
master.exec_stmt("CREATE TABLE test(id INTEGER)")
# It throws an error because the threads are stopped.
self.assertRaises(_errors.DatabaseError, sync_slave_with_master,
slave, master, timeout=0)
# Wait until the slave catches up. It does not throw an error
# because the threads are running.
start_slave(slave, wait=True)
sync_slave_with_master(slave, master, timeout=0)
# This times out because there are no new events.
gtid_executed = "%s:%s" % (str(master.uuid), "1-20")
self.assertRaises(_errors.TimeoutError, wait_for_slave_gtid, slave,
gtid_executed, timeout=3)
def test_check_rpl_health(self):
"""Test check_slave_issues() function.
"""
# Set up replication.
master = self.master
slave = self.slave
slave.disconnect()
# Try to check the health when one cannot connect to the server.
error, ret = check_slave_issues(slave)
self.assertEqual(error, True)
self.assertEqual(ret['is_not_running'], True)
# Try to check the health when change master has not been executed.
slave.connect()
error, ret = check_slave_issues(slave)
self.assertEqual(error, True)
self.assertEqual(ret['is_not_configured'], True)
# Try to check the health after executing change master.
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd
)
error, ret = check_slave_issues(slave)
self.assertEqual(error, True)
self.assertEqual(ret['io_not_running'], True)
self.assertEqual(ret['sql_not_running'], True)
# Try to check the health after starting one thread.
start_slave(slave, wait=True, threads=(SQL_THREAD, ))
error, ret = check_slave_issues(slave)
self.assertEqual(error, True)
self.assertEqual(ret['io_not_running'], True)
# Create data and synchronize to show there is no gtid behind.
master.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
master.exec_stmt("USE test")
master.exec_stmt("DROP TABLE IF EXISTS test")
master.exec_stmt("CREATE TABLE test(id INTEGER)")
start_slave(slave, wait=True, threads=(IO_THREAD, ))
sync_slave_with_master(slave, master, timeout=0)
result = check_slave_delay(slave, master)
expected_result = {
'seconds_behind': 0,
'is_not_configured': False,
'gtids_behind': 0,
'is_not_running': False,
'sql_delay': 0
}
self.assertEqual(result, expected_result)
def test_get_gtid_behind(self):
"""Test get_gtid_behind() function.
"""
# Set up replication.
master = self.master
slave = self.slave
switch_master(slave, master, MySQLInstances().user,
MySQLInstances().passwd
)
# Check gtid that has no information on server_uuid.
self.assertRaises(_errors.ProgrammingError, get_num_gtid, "1")
sid_1 = "80139491-08ed-11e2-b7bd-f0def124dcc5"
sid_2 = "99939491-08ed-11e2-b7bd-f0def124dcc5"
# Check the pattern sid:trx_id.
ret = get_num_gtid("%s:%s" % (sid_1, "5"))
self.assertEqual(ret, 1)
# Check the pattern sid:trx_id-trx_id.
ret = get_num_gtid("%s:%s" % (sid_1, "5-10"))
self.assertEqual(ret, 6)
# Check the pattern sid:trx_id-trx_id, trx_id, trx_id-trx-id.
ret = get_num_gtid("%s:%s,%s,%s" % (sid_1, "5-10", "20", "25-30"))
self.assertEqual(ret, 13)
# Check the pattern sid:trx_id-trx_id, sid:trx_id-trx-id.
ret = get_num_gtid("%s:%s,%s:%s" % (sid_1, "5-10", sid_2, "5-6"))
self.assertEqual(ret, 8)
# Check the pattern sid:trx_id-trx_id, sid:trx_id-trx-id but filtering
# server_uuids that are different from sid_2.
ret = get_num_gtid("%s:%s,%s:%s" %
(sid_1, "5-10", sid_2, "5-6"), sid_2)
self.assertEqual(ret, 2)
# Check empty master_gtid_status and empty slave_gtid_status.
master_gtid_status = master.get_gtid_status()
slave_gtid_status = slave.get_gtid_status()
ret = get_slave_num_gtid_behind(slave, master_gtid_status)
self.assertEqual(ret, 0)
self.assertEqual(slave_gtid_status[0].GTID_EXECUTED, "")
self.assertEqual(master_gtid_status[0].GTID_EXECUTED, "")
# It is not possible to do any comparison if the master_gtid_status
# is empty.
slave.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
slave.exec_stmt("USE test")
slave.exec_stmt("DROP TABLE IF EXISTS test")
master_gtid_status = master.get_gtid_status()
slave_gtid_status = slave.get_gtid_status()
self.assertRaises(_errors.InvalidGtidError, get_slave_num_gtid_behind,
slave, master_gtid_status)
self.assertNotEqual(slave_gtid_status[0].GTID_EXECUTED, "")
self.assertEqual(master_gtid_status[0].GTID_EXECUTED, "")
# Check what happens if there are different sets of transactions.
master.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
master_gtid_status = master.get_gtid_status()
slave_gtid_status = slave.get_gtid_status()
ret = get_slave_num_gtid_behind(slave, master_gtid_status)
self.assertEqual(ret, 1)
self.assertNotEqual(slave_gtid_status[0].GTID_EXECUTED, "")
self.assertNotEqual(master_gtid_status[0].GTID_EXECUTED, "")
# Check what happens if the slave_gtid_status is empty.
reset_master(slave)
master_gtid_status = master.get_gtid_status()
slave_gtid_status = slave.get_gtid_status()
ret = get_slave_num_gtid_behind(slave, master_gtid_status)
self.assertEqual(ret, 1)
self.assertEqual(slave_gtid_status[0].GTID_EXECUTED, "")
self.assertNotEqual(master_gtid_status[0].GTID_EXECUTED, "")
