def configure_servers(options): """Check if some MySQL's addresses were specified and the number is greater than NUMBER_OF_SERVERS. """ import tests.utils as _test_utils from mysql.fabric.server import ( MySQLServer, ConnectionPool, ) try: servers = _test_utils.MySQLInstances() servers.state_store_address = "{host}:{port}".format( host=options.host, port=options.port ) servers.user = options.db_user servers.passwd = None servers.root_user = options.user servers.root_passwd = options.password if options.servers: for address in options.servers.split(): servers.add_address(address) uuid = MySQLServer.discover_uuid( address=address, user=servers.root_user, passwd=servers.root_passwd ) server = MySQLServer( _uuid.UUID(uuid), address=address, user=servers.root_user, passwd=servers.root_passwd ) server.connect() server.set_session_binlog(False) server.exec_stmt( "GRANT {privileges} ON *.* TO '{user}'@'%%'".format( privileges=", ".join(MySQLServer.ALL_PRIVILEGES), user=servers.user) ) server.exec_stmt("FLUSH PRIVILEGES") server.set_session_binlog(True) server.disconnect() ConnectionPool().purge_connections(server.uuid) if servers.get_number_addresses() < NUMBER_OF_SERVERS: print "<<<<<<<<<< Some unit tests need %s MySQL Instances. " \ ">>>>>>>>>> " % (NUMBER_OF_SERVERS, ) return False except Exception as error: print "Error configuring servers:", error return False return True
class TestHashSplitGlobal(tests.utils.TestCase): """Contains unit tests for testing the shard split operation and for verifying that the global server configuration remains constant after the shard split configuration. """ def setUp(self): """Configure the existing environment """ tests.utils.cleanup_environment() self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(0), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__server_1.connect() self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(1), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_2.exec_stmt("CREATE DATABASE db2") self.__server_2.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_2.exec_stmt("CREATE DATABASE db3") self.__server_2.exec_stmt("CREATE TABLE db3.t3" "(userID INT, Department INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_2.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, %s)" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(2), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add(self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_3.exec_stmt("CREATE DATABASE db2") self.__server_3.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_3.exec_stmt("CREATE DATABASE db3") self.__server_3.exec_stmt("CREATE TABLE db3.t3" "(userID INT, Department INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_3.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, %s)" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add(self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(3), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_4.exec_stmt("CREATE DATABASE db2") self.__server_4.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_4.exec_stmt("CREATE DATABASE db3") self.__server_4.exec_stmt("CREATE TABLE db3.t3" "(userID INT, Department INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_4.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, %s)" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add(self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(4), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_5.exec_stmt("CREATE DATABASE db2") self.__server_5.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_5.exec_stmt("CREATE DATABASE db3") self.__server_5.exec_stmt("CREATE TABLE db3.t3" "(userID INT, Department INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_5.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, %s)" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add(self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(5), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add(self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) status = self.proxy.sharding.create_definition("HASH", "GROUPID1") self.check_xmlrpc_command_result(status, returns=1) status = self.proxy.sharding.add_table(1, "db1.t1", "userID") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_table(1, "db2.t2", "userID") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_table(1, "db3.t3", "userID") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_shard( 1, "GROUPID2,GROUPID3,GROUPID4,GROUPID5", "ENABLED") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.prune_shard("db1.t1") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.prune_shard("db2.t2") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.prune_shard("db3.t3") self.check_xmlrpc_command_result(status) def test_split_shard_1(self): '''Test the split of shard 1 and the global server configuration after that. The test splits shard 1 between GROUPID2 and GROUPID6. After the split is done, it verifies the count on GROUPID6 to check that the group has tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. ''' row_cnt_shard_db1_t1 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_db2_t2 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_db3_t3 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch": True}) row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0]) row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0]) row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0]) status = self.proxy.sharding.split_shard("1", "GROUPID6") self.check_xmlrpc_command_result(status) self.__server_6.connect() self.__server_2.connect() row_cnt_shard_after_split_1_db1_t1 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_after_split_1_db2_t2 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_after_split_1_db3_t3 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch": True}) row_cnt_shard_after_split_1_db1_t1_a = \ int(row_cnt_shard_after_split_1_db1_t1[0][0]) row_cnt_shard_after_split_1_db2_t2_a = \ int(row_cnt_shard_after_split_1_db2_t2[0][0]) row_cnt_shard_after_split_1_db3_t3_a = \ int(row_cnt_shard_after_split_1_db3_t3[0][0]) row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch": True}) row_cnt_shard_after_split_1_db1_t1_b = \ int(row_cnt_shard_after_split_1_db1_t1_b[0][0]) row_cnt_shard_after_split_1_db2_t2_b = \ int(row_cnt_shard_after_split_1_db2_t2_b[0][0]) row_cnt_shard_after_split_1_db3_t3_b = \ int(row_cnt_shard_after_split_1_db3_t3_b[0][0]) self.assertTrue( row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a + row_cnt_shard_after_split_1_db1_t1_b)) self.assertTrue( row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a + row_cnt_shard_after_split_1_db2_t2_b)) self.assertTrue( row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a + row_cnt_shard_after_split_1_db3_t3_b)) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_split_shard_2(self): '''Test the split of shard 2 and the global server configuration after that. The test splits shard 2 between GROUPID3 and GROUPID6. After the split is done, it verifies the count on GROUPID6 to check that the group has tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. ''' row_cnt_shard_db1_t1 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_db2_t2 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_db3_t3 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch": True}) row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0]) row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0]) row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0]) status = self.proxy.sharding.split_shard("2", "GROUPID6") self.check_xmlrpc_command_result(status) self.__server_6.connect() self.__server_3.connect() row_cnt_shard_after_split_1_db1_t1 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_after_split_1_db2_t2 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_after_split_1_db3_t3 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch": True}) row_cnt_shard_after_split_1_db1_t1_a = \ int(row_cnt_shard_after_split_1_db1_t1[0][0]) row_cnt_shard_after_split_1_db2_t2_a = \ int(row_cnt_shard_after_split_1_db2_t2[0][0]) row_cnt_shard_after_split_1_db3_t3_a = \ int(row_cnt_shard_after_split_1_db3_t3[0][0]) row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch": True}) row_cnt_shard_after_split_1_db1_t1_b = \ int(row_cnt_shard_after_split_1_db1_t1_b[0][0]) row_cnt_shard_after_split_1_db2_t2_b = \ int(row_cnt_shard_after_split_1_db2_t2_b[0][0]) row_cnt_shard_after_split_1_db3_t3_b = \ int(row_cnt_shard_after_split_1_db3_t3_b[0][0]) self.assertTrue( row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a + row_cnt_shard_after_split_1_db1_t1_b)) self.assertTrue( row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a + row_cnt_shard_after_split_1_db2_t2_b)) self.assertTrue( row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a + row_cnt_shard_after_split_1_db3_t3_b)) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_split_shard_3(self): '''Test the split of shard 3 and the global server configuration after that. The test splits shard 3 between GROUPID4 and GROUPID6. After the split is done, it verifies the count on GROUPID6 to check that the group has tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. ''' row_cnt_shard_db1_t1 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_db2_t2 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_db3_t3 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch": True}) row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0]) row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0]) row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0]) status = self.proxy.sharding.split_shard("3", "GROUPID6") self.check_xmlrpc_command_result(status) self.__server_6.connect() self.__server_4.connect() row_cnt_shard_after_split_1_db1_t1 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_after_split_1_db2_t2 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_after_split_1_db3_t3 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch": True}) row_cnt_shard_after_split_1_db1_t1_a = \ int(row_cnt_shard_after_split_1_db1_t1[0][0]) row_cnt_shard_after_split_1_db2_t2_a = \ int(row_cnt_shard_after_split_1_db2_t2[0][0]) row_cnt_shard_after_split_1_db3_t3_a = \ int(row_cnt_shard_after_split_1_db3_t3[0][0]) row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch": True}) row_cnt_shard_after_split_1_db1_t1_b = \ int(row_cnt_shard_after_split_1_db1_t1_b[0][0]) row_cnt_shard_after_split_1_db2_t2_b = \ int(row_cnt_shard_after_split_1_db2_t2_b[0][0]) row_cnt_shard_after_split_1_db3_t3_b = \ int(row_cnt_shard_after_split_1_db3_t3_b[0][0]) self.assertTrue( row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a + row_cnt_shard_after_split_1_db1_t1_b)) self.assertTrue( row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a + row_cnt_shard_after_split_1_db2_t2_b)) self.assertTrue( row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a + row_cnt_shard_after_split_1_db3_t3_b)) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_split_shard_4(self): '''Test the split of shard 4 and the global server configuration after that. The test splits shard 4 between GROUPID5 and GROUPID6. After the split is done, it verifies the count on GROUPID6 to check that the group has tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. ''' row_cnt_shard_db1_t1 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_db2_t2 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_db3_t3 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch": True}) row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0]) row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0]) row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0]) status = self.proxy.sharding.split_shard("4", "GROUPID6") self.check_xmlrpc_command_result(status) self.__server_6.connect() self.__server_5.connect() row_cnt_shard_after_split_1_db1_t1 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_after_split_1_db2_t2 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_after_split_1_db3_t3 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch": True}) row_cnt_shard_after_split_1_db1_t1_a = \ int(row_cnt_shard_after_split_1_db1_t1[0][0]) row_cnt_shard_after_split_1_db2_t2_a = \ int(row_cnt_shard_after_split_1_db2_t2[0][0]) row_cnt_shard_after_split_1_db3_t3_a = \ int(row_cnt_shard_after_split_1_db3_t3[0][0]) row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch": True}) row_cnt_shard_after_split_1_db1_t1_b = \ int(row_cnt_shard_after_split_1_db1_t1_b[0][0]) row_cnt_shard_after_split_1_db2_t2_b = \ int(row_cnt_shard_after_split_1_db2_t2_b[0][0]) row_cnt_shard_after_split_1_db3_t3_b = \ int(row_cnt_shard_after_split_1_db3_t3_b[0][0]) self.assertTrue( row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a + row_cnt_shard_after_split_1_db1_t1_b)) self.assertTrue( row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a + row_cnt_shard_after_split_1_db2_t2_b)) self.assertTrue( row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a + row_cnt_shard_after_split_1_db3_t3_b)) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_hash_dump(self): """Test the dump of the HASH sharding information for the table. """ shardinginformation_1 = tests.utils.make_info_result( [[ 'db1', 't1', 'userID', 'E2996A7B8509367020B55A4ACD2AE46A', '1', 'HASH', 'GROUPID2', 'GROUPID1' ], [ 'db1', 't1', 'userID', 'E88F547DF45C99F27646C76316FB21DF', '2', 'HASH', 'GROUPID3', 'GROUPID1' ], [ 'db1', 't1', 'userID', '7A2E76448FF04233F3851A492BEF1090', '3', 'HASH', 'GROUPID4', 'GROUPID1' ], [ 'db1', 't1', 'userID', '97427AA63E300F56536710F5D73A35FA', '4', 'HASH', 'GROUPID5', 'GROUPID1' ]]) packet = self.proxy.dump.sharding_information(0, "db1.t1") self.check_xmlrpc_result(packet, shardinginformation_1) def tearDown(self): """Clean up the existing environment """ tests.utils.cleanup_environment()
def test_privileges(self): """Test whether user's have the appropriate privileges. """ # Some privileges MINIMUM_PRIVILEGES = [ "REPLICATION SLAVE", "REPLICATION CLIENT", "SUPER", "SHOW DATABASES", "RELOAD" ] # Connect to server as admin and create temporary user. uuid = MySQLServer.discover_uuid(OPTIONS["address"]) server = MySQLServer(_uuid.UUID(uuid), OPTIONS["address"], tests.utils.MySQLInstances().user, tests.utils.MySQLInstances().passwd) ConnectionManager().purge_connections(server) server.connect() server.set_session_binlog(False) server.exec_stmt("CREATE USER 'jeffrey'@'%%' IDENTIFIED BY 'mypass'") # Check if jeffrey (temporary user) has the appropriate privileges. # There is not privilege associate to jeffrey. new_server = MySQLServer(_uuid.UUID(uuid), OPTIONS["address"], "jeffrey", "mypass") new_server.connect() self.assertFalse(new_server.has_privileges(MINIMUM_PRIVILEGES)) # Check if jeffrey (temporary user) has the appropriate privileges. # Grant required privileges except RELOAD # There is no RELOAD on a global level. privileges = ", ".join( [priv for priv in MINIMUM_PRIVILEGES if priv != "RELOAD"]) server.exec_stmt("GRANT {privileges} ON *.* TO 'jeffrey'@'%%'".format( privileges=privileges)) self.assertFalse(new_server.has_privileges(MINIMUM_PRIVILEGES)) # Check if jeffrey (temporary user) has the appropriate privileges. # The RELOAD on a global level was granted. server.exec_stmt("GRANT RELOAD ON *.* TO 'jeffrey'@'%%'") self.assertTrue(new_server.has_privileges(MINIMUM_PRIVILEGES)) # Check if jeffrey (temporary user) has the appropriate privileges. # Revoke privilegs from temporary user. # There is no ALL on a global level. server.exec_stmt("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " "'jeffrey'@'%%'") server.exec_stmt("GRANT ALL ON fabric.* TO 'jeffrey'@'%%'") self.assertFalse(new_server.has_privileges(MINIMUM_PRIVILEGES)) # Check if jeffrey (temporary user) has the appropriate privileges. # Grant all privileges, which the administrative user has to have. server.exec_stmt("GRANT" " ALTER, ALTER ROUTINE, CREATE, CREATE ROUTINE," " CREATE TEMPORARY TABLES, CREATE USER," " CREATE VIEW, DELETE, DROP, EVENT, EXECUTE," " GRANT OPTION, INDEX, INSERT, LOCK TABLES, PROCESS, " " RELOAD, REPLICATION CLIENT, REPLICATION SLAVE," " SELECT, SHOW DATABASES, SHOW VIEW, SHUTDOWN," " SUPER, TRIGGER, UPDATE" " ON *.* TO 'jeffrey'@'%%'") self.assertTrue(new_server.has_privileges(MINIMUM_PRIVILEGES)) # Drop temporary user. server.exec_stmt("DROP USER 'jeffrey'@'%%'") server.set_session_binlog(True) server.disconnect() new_server.disconnect() ConnectionManager().purge_connections(server)
class TestHashSharding(tests.utils.TestCase): def setUp(self): self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(0), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(1), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(2), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add(self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add(self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(3), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add(self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(4), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add(self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(5), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__server_6.connect() self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_6.exec_stmt("CREATE DATABASE db1") self.__server_6.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add(self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn() self.assertEquals(self.__shard_mapping_list, []) self.__shard_mapping_id_1 = ShardMapping.define("HASH", "GROUPID1") self.__shard_mapping_1 = ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID") self.__shard_1 = Shards.add("GROUPID2") self.__shard_2 = Shards.add("GROUPID3") self.__shard_3 = Shards.add("GROUPID4") self.__shard_4 = Shards.add("GROUPID5") self.__shard_5 = Shards.add("GROUPID6") self.__hash_sharding_specification_1 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_1.shard_id) self.__hash_sharding_specification_2 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_2.shard_id) self.__hash_sharding_specification_3 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_3.shard_id) self.__hash_sharding_specification_4 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_4.shard_id) self.__hash_sharding_specification_5 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_5.shard_id) def test_hash_lookup(self): """Test the hash sharding lookup. """ shard_1_cnt = 0 shard_2_cnt = 0 shard_3_cnt = 0 shard_4_cnt = 0 shard_5_cnt = 0 #Lookup a range of keys to ensure that all the shards are #utilized. for i in range(0, 1000): hash_sharding_spec_1 = HashShardingSpecification.lookup( i, self.__shard_mapping_id_1, "HASH") if self.__shard_1.shard_id == hash_sharding_spec_1.shard_id: shard_1_cnt = shard_1_cnt + 1 elif self.__shard_2.shard_id == hash_sharding_spec_1.shard_id: shard_2_cnt = shard_2_cnt + 1 elif self.__shard_3.shard_id == hash_sharding_spec_1.shard_id: shard_3_cnt = shard_3_cnt + 1 elif self.__shard_4.shard_id == hash_sharding_spec_1.shard_id: shard_4_cnt = shard_4_cnt + 1 elif self.__shard_5.shard_id == hash_sharding_spec_1.shard_id: shard_5_cnt = shard_5_cnt + 1 #The following will ensure that both the hash shards are utilized #to store the keys and the values are not skewed in one shard. self.assertTrue(shard_1_cnt > 0) self.assertTrue(shard_2_cnt > 0) self.assertTrue(shard_3_cnt > 0) self.assertTrue(shard_4_cnt > 0) self.assertTrue(shard_5_cnt > 0) def test_hash_remove(self): """Test the removal of hash shards. """ hash_sharding_specification_1 = HashShardingSpecification.fetch(1) hash_sharding_specification_2 = HashShardingSpecification.fetch(2) hash_sharding_specification_3 = HashShardingSpecification.fetch(3) hash_sharding_specification_4 = HashShardingSpecification.fetch(4) hash_sharding_specification_5 = HashShardingSpecification.fetch(5) hash_sharding_specification_1.remove() hash_sharding_specification_2.remove() hash_sharding_specification_3.remove() hash_sharding_specification_4.remove() hash_sharding_specification_5.remove() self.__shard_1.remove() self.__shard_2.remove() self.__shard_3.remove() self.__shard_4.remove() self.__shard_5.remove() for i in range(0, 10): hash_sharding_spec = HashShardingSpecification.lookup( i, self.__shard_mapping_id_1, "HASH") self.assertEqual(hash_sharding_spec, None) def test_list_shard_mapping(self): """Test the listing of HASH shards in a shard mapping. """ expected_shard_mapping_list1 = [1, "HASH", "GROUPID1"] obtained_shard_mapping_list = ShardMapping.list_shard_mapping_defn() self.assertEqual(set(expected_shard_mapping_list1), set(obtained_shard_mapping_list[0])) def test_shard_mapping_list_mappings(self): """Test the listing of all HASH shards in a shard mapping. """ shard_mappings = ShardMapping.list("HASH") self.assertTrue( ShardingUtils.compare_shard_mapping(self.__shard_mapping_1, shard_mappings[0])) def test_fetch_sharding_scheme(self): """Test the fetch method of the HASH sharding scheme. """ hash_sharding_specification_1 = HashShardingSpecification.fetch(1) hash_sharding_specification_2 = HashShardingSpecification.fetch(2) hash_sharding_specification_3 = HashShardingSpecification.fetch(3) hash_sharding_specification_4 = HashShardingSpecification.fetch(4) hash_sharding_specification_5 = HashShardingSpecification.fetch(5) hash_sharding_specifications = HashShardingSpecification.list(1) #list does not return the hashing specifications in order of shard_id, #hence a direct comparison is not posssible. self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_1, hash_sharding_specifications)) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_2, hash_sharding_specifications)) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_3, hash_sharding_specifications)) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_4, hash_sharding_specifications)) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_5, hash_sharding_specifications)) def test_prune_shard(self): rows = self.__server_2.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 500) rows = self.__server_3.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 500) rows = self.__server_4.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 500) rows = self.__server_5.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 500) rows = self.__server_6.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 500) status = self.proxy.sharding.prune_shard("db1.t1") self.check_xmlrpc_command_result(status) rows = self.__server_2.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) cnt1 = int(rows[0][0]) self.assertTrue(int(rows[0][0]) < 500) rows = self.__server_3.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) cnt2 = int(rows[0][0]) self.assertTrue(int(rows[0][0]) < 500) rows = self.__server_4.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) cnt3 = int(rows[0][0]) self.assertTrue(int(rows[0][0]) < 500) rows = self.__server_5.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) cnt4 = int(rows[0][0]) self.assertTrue(int(rows[0][0]) < 500) rows = self.__server_6.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) cnt5 = int(rows[0][0]) self.assertTrue(int(rows[0][0]) < 500) self.assertTrue((cnt1 + cnt2 + cnt3 + cnt4 + cnt5) == 500) def test_prune_lookup(self): self.proxy.sharding.prune_shard("db1.t1") rows = self.__server_2.exec_stmt("SELECT userID FROM db1.t1", {"fetch": True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_1 = HashShardingSpecification.lookup( val, self.__shard_mapping_id_1, "HASH") self.assertEqual(hash_sharding_spec_1.shard_id, 1) rows = self.__server_3.exec_stmt("SELECT userID FROM db1.t1", {"fetch": True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_2 = HashShardingSpecification.lookup( val, self.__shard_mapping_id_1, "HASH") self.assertEqual(hash_sharding_spec_2.shard_id, 2) rows = self.__server_4.exec_stmt("SELECT userID FROM db1.t1", {"fetch": True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_3 = HashShardingSpecification.lookup( val, self.__shard_mapping_id_1, "HASH") self.assertEqual(hash_sharding_spec_3.shard_id, 3) rows = self.__server_5.exec_stmt("SELECT userID FROM db1.t1", {"fetch": True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_4 = HashShardingSpecification.lookup( val, self.__shard_mapping_id_1, "HASH") self.assertEqual(hash_sharding_spec_4.shard_id, 4) rows = self.__server_6.exec_stmt("SELECT userID FROM db1.t1", {"fetch": True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_5 = HashShardingSpecification.lookup( val, self.__shard_mapping_id_1, "HASH") self.assertEqual(hash_sharding_spec_5.shard_id, 5) def hash_sharding_specification_in_list(self, hash_sharding_spec, hash_sharding_specification_list): """Verify if the given hash sharding specification is present in the list of hash sharding specifications. :param hash_sharding_spec: The hash sharding specification that needs to be lookedup. :param hash_sharding_specification_list: The list of hash sharding specifications :return: True if the hash sharding specification is present. False otherwise. """ for i in range(0, len(hash_sharding_specification_list)): if ShardingUtils.compare_hash_specifications( hash_sharding_spec, hash_sharding_specification_list[i]): return True return False def tearDown(self): """Tear down the state store setup. """ tests.utils.cleanup_environment()
class TestShardingPrune(unittest.TestCase): def assertStatus(self, status, expect): items = (item['diagnosis'] for item in status[1] if item['diagnosis']) self.assertEqual(status[1][-1]["success"], expect, "\n".join(items)) def setUp(self): """Creates the following topology for testing, GROUPID1 - localhost:13001, localhost:13002 - Global Group GROUPID2 - localhost:13003, localhost:13004 - shard 1 GROUPID3 - localhost:13005, localhost:13006 - shard 2 """ self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 601): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add( self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 601): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add( self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(3), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 601): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add( self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(4), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 601): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add( self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(5), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__server_6.connect() self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_6.exec_stmt("CREATE DATABASE db1") self.__server_6.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 601): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add( self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) status = self.proxy.sharding.create_definition("RANGE", "GROUPID1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_define_shard_mapping).") self.assertEqual(status[2], 1) status = self.proxy.sharding.add_table(1, "db1.t1", "userID") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_shard( 1, "GROUPID2/1,GROUPID3/101,GROUPID4/201," "GROUPID5/301,GROUPID6/401", "ENABLED" ) self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard).") def test_prune_shard(self): status = self.proxy.sharding.prune_shard("db1.t1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables).") status = self.proxy.sharding.lookup_servers("db1.t1", 1, "LOCAL") self.assertEqual(status[0], True) self.assertEqual(status[1], "") obtained_server_list = status[2] shard_uuid = obtained_server_list[0][0] shard_server = MySQLServer.fetch(shard_uuid) shard_server.connect() rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 100) rows = shard_server.exec_stmt( "SELECT MAX(userID) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 100) rows = shard_server.exec_stmt( "SELECT MIN(userID) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 1) status = self.proxy.sharding.lookup_servers("db1.t1", 101, "LOCAL") self.assertEqual(status[0], True) self.assertEqual(status[1], "") obtained_server_list = status[2] shard_uuid = obtained_server_list[0][0] shard_server = MySQLServer.fetch(shard_uuid) shard_server.connect() rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 100) rows = shard_server.exec_stmt( "SELECT MAX(userID) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 200) rows = shard_server.exec_stmt( "SELECT MIN(userID) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 101) status = self.proxy.sharding.lookup_servers("db1.t1", 202, "LOCAL") self.assertEqual(status[0], True) self.assertEqual(status[1], "") obtained_server_list = status[2] shard_uuid = obtained_server_list[0][0] shard_server = MySQLServer.fetch(shard_uuid) shard_server.connect() rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 100) rows = shard_server.exec_stmt( "SELECT MAX(userID) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 300) rows = shard_server.exec_stmt( "SELECT MIN(userID) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 201) status = self.proxy.sharding.lookup_servers("db1.t1", 303, "LOCAL") self.assertEqual(status[0], True) self.assertEqual(status[1], "") obtained_server_list = status[2] shard_uuid = obtained_server_list[0][0] shard_server = MySQLServer.fetch(shard_uuid) shard_server.connect() rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 100) rows = shard_server.exec_stmt( "SELECT MAX(userID) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 400) rows = shard_server.exec_stmt( "SELECT MIN(userID) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 301) status = self.proxy.sharding.lookup_servers("db1.t1", 404, "LOCAL") self.assertEqual(status[0], True) self.assertEqual(status[1], "") obtained_server_list = status[2] shard_uuid = obtained_server_list[0][0] shard_server = MySQLServer.fetch(shard_uuid) shard_server.connect() rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 200) rows = shard_server.exec_stmt( "SELECT MAX(userID) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 600) rows = shard_server.exec_stmt( "SELECT MIN(userID) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 401) def tearDown(self): self.__server_2.exec_stmt("DROP TABLE db1.t1") self.__server_2.exec_stmt("DROP DATABASE db1") self.__server_3.exec_stmt("DROP TABLE db1.t1") self.__server_3.exec_stmt("DROP DATABASE db1") self.__server_4.exec_stmt("DROP TABLE db1.t1") self.__server_4.exec_stmt("DROP DATABASE db1") self.__server_5.exec_stmt("DROP TABLE db1.t1") self.__server_5.exec_stmt("DROP DATABASE db1") self.__server_6.exec_stmt("DROP TABLE db1.t1") self.__server_6.exec_stmt("DROP DATABASE db1") status = self.proxy.sharding.disable_shard("1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_disable_shard).") status = self.proxy.sharding.disable_shard("2") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_disable_shard).") status = self.proxy.sharding.disable_shard("3") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_disable_shard).") status = self.proxy.sharding.disable_shard("4") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_disable_shard).") status = self.proxy.sharding.disable_shard("5") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_disable_shard).") status = self.proxy.sharding.remove_shard("1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_remove_shard).") status = self.proxy.sharding.remove_shard("2") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_remove_shard).") status = self.proxy.sharding.remove_shard("3") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_remove_shard).") status = self.proxy.sharding.remove_shard("4") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_remove_shard).") status = self.proxy.sharding.remove_shard("5") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_remove_shard).") status = self.proxy.sharding.remove_table("db1.t1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_remove_shard_mapping).") status = self.proxy.sharding.remove_definition("1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_remove_shard_mapping_defn).") self.proxy.group.demote("GROUPID1") self.proxy.group.demote("GROUPID2") self.proxy.group.demote("GROUPID3") self.proxy.group.demote("GROUPID4") self.proxy.group.demote("GROUPID5") self.proxy.group.demote("GROUPID6") for group_id in ("GROUPID1", "GROUPID2", "GROUPID3", "GROUPID4", "GROUPID5", "GROUPID6"): status = self.proxy.group.lookup_servers(group_id) self.assertEqual(status[0], True) self.assertEqual(status[1], "") obtained_server_list = status[2] status = \ self.proxy.group.remove( group_id, obtained_server_list[0]["server_uuid"] ) self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_remove_server).") status = self.proxy.group.destroy(group_id) self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_destroy_group).") tests.utils.cleanup_environment() tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestHashSplitGlobal(unittest.TestCase): """Contains unit tests for testing the shard split operation and for verifying that the global server configuration remains constant after the shard split configuration. """ def assertStatus(self, status, expect): items = (item['diagnosis'] for item in status[1] if item['diagnosis']) self.assertEqual(status[1][-1]["success"], expect, "\n".join(items)) def setUp(self): """Configure the existing environment """ tests.utils.cleanup_environment() self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__server_1.connect() self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_2.exec_stmt("CREATE DATABASE db2") self.__server_2.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_2.exec_stmt("CREATE DATABASE db3") self.__server_2.exec_stmt("CREATE TABLE db3.t3" "(userID INT, Department INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_2.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, %s)" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add( self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_3.exec_stmt("CREATE DATABASE db2") self.__server_3.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_3.exec_stmt("CREATE DATABASE db3") self.__server_3.exec_stmt("CREATE TABLE db3.t3" "(userID INT, Department INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_3.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, %s)" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add( self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(3), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_4.exec_stmt("CREATE DATABASE db2") self.__server_4.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_4.exec_stmt("CREATE DATABASE db3") self.__server_4.exec_stmt("CREATE TABLE db3.t3" "(userID INT, Department INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_4.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, %s)" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add( self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(4), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_5.exec_stmt("CREATE DATABASE db2") self.__server_5.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_5.exec_stmt("CREATE DATABASE db3") self.__server_5.exec_stmt("CREATE TABLE db3.t3" "(userID INT, Department INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_5.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, %s)" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add( self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(5), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add( self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) status = self.proxy.sharding.create_definition("HASH", "GROUPID1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_define_shard_mapping).") self.assertEqual(status[2], 1) status = self.proxy.sharding.add_table(1, "db1.t1", "userID") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_table(1, "db2.t2", "userID") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_table(1, "db3.t3", "userID") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_shard(1, "GROUPID2,GROUPID3,GROUPID4,GROUPID5", "ENABLED") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard).") status = self.proxy.sharding.prune_shard("db1.t1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables).") status = self.proxy.sharding.prune_shard("db2.t2") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables).") status = self.proxy.sharding.prune_shard("db3.t3") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables).") def test_split_shard_1(self): '''Test the split of shard 1 and the global server configuration after that. The test splits shard 1 between GROUPID2 and GROUPID6. After the split is done, it verifies the count on GROUPID6 to check that the group has tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. ''' row_cnt_shard_db1_t1 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_db2_t2 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_db3_t3 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch" : True} ) row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0]) row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0]) row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0]) status = self.proxy.sharding.split_shard("1", "GROUPID6") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables_after_split).") self.__server_6.connect() row_cnt_shard_after_split_1_db1_t1 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_1_db2_t2 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_1_db3_t3 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch" : True} ) row_cnt_shard_after_split_1_db1_t1_a = \ int(row_cnt_shard_after_split_1_db1_t1[0][0]) row_cnt_shard_after_split_1_db2_t2_a = \ int(row_cnt_shard_after_split_1_db2_t2[0][0]) row_cnt_shard_after_split_1_db3_t3_a = \ int(row_cnt_shard_after_split_1_db3_t3[0][0]) row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch" : True} ) row_cnt_shard_after_split_1_db1_t1_b = \ int(row_cnt_shard_after_split_1_db1_t1_b[0][0]) row_cnt_shard_after_split_1_db2_t2_b = \ int(row_cnt_shard_after_split_1_db2_t2_b[0][0]) row_cnt_shard_after_split_1_db3_t3_b = \ int(row_cnt_shard_after_split_1_db3_t3_b[0][0]) self.assertTrue( row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a + row_cnt_shard_after_split_1_db1_t1_b) ) self.assertTrue( row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a + row_cnt_shard_after_split_1_db2_t2_b) ) self.assertTrue( row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a + row_cnt_shard_after_split_1_db3_t3_b) ) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_split_shard_2(self): '''Test the split of shard 2 and the global server configuration after that. The test splits shard 2 between GROUPID3 and GROUPID6. After the split is done, it verifies the count on GROUPID6 to check that the group has tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. ''' row_cnt_shard_db1_t1 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_db2_t2 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_db3_t3 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch" : True} ) row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0]) row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0]) row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0]) status = self.proxy.sharding.split_shard("2", "GROUPID6") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables_after_split).") self.__server_6.connect() row_cnt_shard_after_split_1_db1_t1 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_1_db2_t2 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_1_db3_t3 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch" : True} ) row_cnt_shard_after_split_1_db1_t1_a = \ int(row_cnt_shard_after_split_1_db1_t1[0][0]) row_cnt_shard_after_split_1_db2_t2_a = \ int(row_cnt_shard_after_split_1_db2_t2[0][0]) row_cnt_shard_after_split_1_db3_t3_a = \ int(row_cnt_shard_after_split_1_db3_t3[0][0]) row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch" : True} ) row_cnt_shard_after_split_1_db1_t1_b = \ int(row_cnt_shard_after_split_1_db1_t1_b[0][0]) row_cnt_shard_after_split_1_db2_t2_b = \ int(row_cnt_shard_after_split_1_db2_t2_b[0][0]) row_cnt_shard_after_split_1_db3_t3_b = \ int(row_cnt_shard_after_split_1_db3_t3_b[0][0]) self.assertTrue( row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a + row_cnt_shard_after_split_1_db1_t1_b) ) self.assertTrue( row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a + row_cnt_shard_after_split_1_db2_t2_b) ) self.assertTrue( row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a + row_cnt_shard_after_split_1_db3_t3_b) ) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_split_shard_3(self): '''Test the split of shard 3 and the global server configuration after that. The test splits shard 3 between GROUPID4 and GROUPID6. After the split is done, it verifies the count on GROUPID6 to check that the group has tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. ''' row_cnt_shard_db1_t1 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_db2_t2 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_db3_t3 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch" : True} ) row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0]) row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0]) row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0]) status = self.proxy.sharding.split_shard("3", "GROUPID6") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables_after_split).") self.__server_6.connect() row_cnt_shard_after_split_1_db1_t1 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_1_db2_t2 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_1_db3_t3 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch" : True} ) row_cnt_shard_after_split_1_db1_t1_a = \ int(row_cnt_shard_after_split_1_db1_t1[0][0]) row_cnt_shard_after_split_1_db2_t2_a = \ int(row_cnt_shard_after_split_1_db2_t2[0][0]) row_cnt_shard_after_split_1_db3_t3_a = \ int(row_cnt_shard_after_split_1_db3_t3[0][0]) row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch" : True} ) row_cnt_shard_after_split_1_db1_t1_b = \ int(row_cnt_shard_after_split_1_db1_t1_b[0][0]) row_cnt_shard_after_split_1_db2_t2_b = \ int(row_cnt_shard_after_split_1_db2_t2_b[0][0]) row_cnt_shard_after_split_1_db3_t3_b = \ int(row_cnt_shard_after_split_1_db3_t3_b[0][0]) self.assertTrue( row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a + row_cnt_shard_after_split_1_db1_t1_b) ) self.assertTrue( row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a + row_cnt_shard_after_split_1_db2_t2_b) ) self.assertTrue( row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a + row_cnt_shard_after_split_1_db3_t3_b) ) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_split_shard_4(self): '''Test the split of shard 4 and the global server configuration after that. The test splits shard 4 between GROUPID5 and GROUPID6. After the split is done, it verifies the count on GROUPID6 to check that the group has tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. ''' row_cnt_shard_db1_t1 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_db2_t2 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_db3_t3 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch" : True} ) row_cnt_shard_db1_t1 = int(row_cnt_shard_db1_t1[0][0]) row_cnt_shard_db2_t2 = int(row_cnt_shard_db2_t2[0][0]) row_cnt_shard_db3_t3 = int(row_cnt_shard_db3_t3[0][0]) status = self.proxy.sharding.split_shard("4", "GROUPID6") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables_after_split).") self.__server_6.connect() row_cnt_shard_after_split_1_db1_t1 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_1_db2_t2 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_1_db3_t3 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch" : True} ) row_cnt_shard_after_split_1_db1_t1_a = \ int(row_cnt_shard_after_split_1_db1_t1[0][0]) row_cnt_shard_after_split_1_db2_t2_a = \ int(row_cnt_shard_after_split_1_db2_t2[0][0]) row_cnt_shard_after_split_1_db3_t3_a = \ int(row_cnt_shard_after_split_1_db3_t3[0][0]) row_cnt_shard_after_split_1_db1_t1_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_1_db2_t2_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_1_db3_t3_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db3.t3", {"fetch" : True} ) row_cnt_shard_after_split_1_db1_t1_b = \ int(row_cnt_shard_after_split_1_db1_t1_b[0][0]) row_cnt_shard_after_split_1_db2_t2_b = \ int(row_cnt_shard_after_split_1_db2_t2_b[0][0]) row_cnt_shard_after_split_1_db3_t3_b = \ int(row_cnt_shard_after_split_1_db3_t3_b[0][0]) self.assertTrue( row_cnt_shard_db1_t1 == (row_cnt_shard_after_split_1_db1_t1_a + row_cnt_shard_after_split_1_db1_t1_b) ) self.assertTrue( row_cnt_shard_db2_t2 == (row_cnt_shard_after_split_1_db2_t2_a + row_cnt_shard_after_split_1_db2_t2_b) ) self.assertTrue( row_cnt_shard_db3_t3 == (row_cnt_shard_after_split_1_db3_t3_a + row_cnt_shard_after_split_1_db3_t3_b) ) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_hash_dump(self): """Test the dump of the HASH sharding information for the table. """ shardinginformation_1 = [0, 0, 0, [['db1', 't1', 'userID', 'E2996A7B8509367020B55A4ACD2AE46A', '1', 'HASH', 'GROUPID2', 'GROUPID1'], ['db1', 't1', 'userID', 'E88F547DF45C99F27646C76316FB21DF', '2', 'HASH', 'GROUPID3', 'GROUPID1'], ['db1', 't1', 'userID', '7A2E76448FF04233F3851A492BEF1090', '3', 'HASH', 'GROUPID4', 'GROUPID1'], ['db1', 't1', 'userID', '97427AA63E300F56536710F5D73A35FA', '4', 'HASH', 'GROUPID5', 'GROUPID1']]] self.assertEqual( self.proxy.dump.sharding_information(0, "db1.t1"), shardinginformation_1 ) def tearDown(self): """Clean up the existing environment """ self.proxy.sharding.disable_shard(1) self.proxy.sharding.remove_shard(1) self.proxy.sharding.disable_shard(2) self.proxy.sharding.remove_shard(2) self.proxy.sharding.disable_shard(3) self.proxy.sharding.remove_shard(3) self.proxy.sharding.disable_shard(4) self.proxy.sharding.remove_shard(4) self.proxy.sharding.disable_shard(5) self.proxy.sharding.remove_shard(5) tests.utils.cleanup_environment() tests.utils.teardown_xmlrpc(self.manager, self.proxy)
def configure_servers(options, config): """Check if some MySQL's addresses were specified and the number is greater than NUMBER_OF_SERVERS. """ import tests.utils as _test_utils from mysql.fabric.server import ( MySQLServer, ConnectionManager, ) from mysql.fabric.backup import ( MySQLDump, ) try: servers = _test_utils.MySQLInstances() # The administrative user as given in --user and --password options. # In simple use cases "root" is used. servers.user = options.user servers.passwd = options.password # Backing store - "fabric_store/storepw". servers.state_store_address = config.get("storage", "address") servers.store_user = config.get("storage", "user") servers.store_passwd = config.get("storage", "password") servers.store_db = config.get("storage", "database") # Server user - "fabric_server/serverpw". servers.server_user = config.get("servers", "user") servers.server_passwd = config.get("servers", "password") # Backup user - "fabric_backup/backuppw". servers.backup_user = config.get("servers", "backup_user") servers.backup_passwd = config.get("servers", "backup_password") # Restore user - "fabric_restore/restorepw". servers.restore_user = config.get("servers", "restore_user") servers.restore_passwd = config.get("servers", "restore_password") # Set up the backing store. from mysql.fabric import persistence uuid = MySQLServer.discover_uuid( address=servers.state_store_address, user=servers.user, passwd=servers.passwd ) server = MySQLServer( _uuid.UUID(uuid), address=servers.state_store_address, user=servers.user, passwd=servers.passwd ) server.connect() # Precautionary cleanup. server.exec_stmt("DROP DATABASE IF EXISTS %s" % (servers.store_db,)) # Create store user. _test_utils.create_test_user( server, servers.store_user, servers.store_passwd, [(persistence.required_privileges(), "{db}.*".format(db=servers.store_db))] ) # Set up managed servers. if options.servers: for address in options.servers.split(): servers.add_address(address) uuid = MySQLServer.discover_uuid( address=address, user=servers.user, passwd=servers.passwd ) server = MySQLServer( _uuid.UUID(uuid), address=address, user=servers.user, passwd=servers.passwd ) server.connect() server.set_session_binlog(False) server.read_only = False # Drop user databases server.set_foreign_key_checks(False) databases = server.exec_stmt("SHOW DATABASES") for database in databases: if database[0] not in MySQLServer.NO_USER_DATABASES: server.exec_stmt("DROP DATABASE IF EXISTS %s" % (database[0],)) server.set_foreign_key_checks(True) # Create server user. _test_utils.create_test_user( server, servers.server_user, servers.server_passwd, [(MySQLServer.SERVER_PRIVILEGES, "*.*"), (MySQLServer.SERVER_PRIVILEGES_DB, "mysql_fabric.*")] ) # Create backup user. _test_utils.create_test_user( server, servers.backup_user, servers.backup_passwd, [(MySQLDump.BACKUP_PRIVILEGES, "*.*")] ) # Create restore user. _test_utils.create_test_user( server, servers.restore_user, servers.restore_passwd, [(MySQLDump.RESTORE_PRIVILEGES, "*.*")] ) server.set_session_binlog(True) server.disconnect() ConnectionManager().purge_connections(server) if servers.get_number_addresses() < NUMBER_OF_SERVERS: sys.stderr.write( "<<<<<<<<<< Some unit tests need {0} MySQL " \ "Instances. >>>>>>>>>>\n".format(NUMBER_OF_SERVERS) ) return False except Exception as error: sys.stderr.write( "Error configuring servers: {0}\n".format(str(error)) ) import traceback traceback.print_exc() return False return True
class TestHashSharding(unittest.TestCase): def assertStatus(self, status, expect): items = (item['diagnosis'] for item in status[1] if item['diagnosis']) self.assertEqual(status[1][-1]["success"], expect, "\n".join(items)) def setUp(self): self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add( self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add( self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(3), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add( self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(4), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add( self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(5), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__server_6.connect() self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_6.exec_stmt("CREATE DATABASE db1") self.__server_6.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add( self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn() self.assertEquals( self.__shard_mapping_list, []) self.__shard_mapping_id_1 = ShardMapping.define("HASH", "GROUPID1") self.__shard_mapping_1 = ShardMapping.add( self.__shard_mapping_id_1, "db1.t1", "userID" ) self.__shard_1 = Shards.add("GROUPID2") self.__shard_2 = Shards.add("GROUPID3") self.__shard_3 = Shards.add("GROUPID4") self.__shard_4 = Shards.add("GROUPID5") self.__shard_5 = Shards.add("GROUPID6") self.__hash_sharding_specification_1 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_1.shard_id ) self.__hash_sharding_specification_2 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_2.shard_id ) self.__hash_sharding_specification_3 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_3.shard_id ) self.__hash_sharding_specification_4 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_4.shard_id ) self.__hash_sharding_specification_5 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_5.shard_id ) def test_hash_lookup(self): """Test the hash sharding lookup. """ shard_1_cnt = 0 shard_2_cnt = 0 shard_3_cnt = 0 shard_4_cnt = 0 shard_5_cnt = 0 #Lookup a range of keys to ensure that all the shards are #utilized. for i in range(0, 1000): hash_sharding_spec_1 = HashShardingSpecification.lookup( i, self.__shard_mapping_id_1, "HASH" ) if self.__shard_1.shard_id == hash_sharding_spec_1.shard_id: shard_1_cnt = shard_1_cnt + 1 elif self.__shard_2.shard_id == hash_sharding_spec_1.shard_id: shard_2_cnt = shard_2_cnt + 1 elif self.__shard_3.shard_id == hash_sharding_spec_1.shard_id: shard_3_cnt = shard_3_cnt + 1 elif self.__shard_4.shard_id == hash_sharding_spec_1.shard_id: shard_4_cnt = shard_4_cnt + 1 elif self.__shard_5.shard_id == hash_sharding_spec_1.shard_id: shard_5_cnt = shard_5_cnt + 1 #The following will ensure that both the hash shards are utilized #to store the keys and the values are not skewed in one shard. self.assertTrue(shard_1_cnt > 0) self.assertTrue(shard_2_cnt > 0) self.assertTrue(shard_3_cnt > 0) self.assertTrue(shard_4_cnt > 0) self.assertTrue(shard_5_cnt > 0) def test_hash_remove(self): """Test the removal of hash shards. """ hash_sharding_specification_1 = HashShardingSpecification.fetch(1) hash_sharding_specification_2 = HashShardingSpecification.fetch(2) hash_sharding_specification_3 = HashShardingSpecification.fetch(3) hash_sharding_specification_4 = HashShardingSpecification.fetch(4) hash_sharding_specification_5 = HashShardingSpecification.fetch(5) hash_sharding_specification_1.remove() hash_sharding_specification_2.remove() hash_sharding_specification_3.remove() hash_sharding_specification_4.remove() hash_sharding_specification_5.remove() self.__shard_1.remove() self.__shard_2.remove() self.__shard_3.remove() self.__shard_4.remove() self.__shard_5.remove() for i in range(0, 10): hash_sharding_spec = HashShardingSpecification.lookup( i, self.__shard_mapping_id_1, "HASH" ) self.assertEqual(hash_sharding_spec, None) def test_list_shard_mapping(self): """Test the listing of HASH shards in a shard mapping. """ expected_shard_mapping_list1 = [1, "HASH", "GROUPID1"] obtained_shard_mapping_list = ShardMapping.list_shard_mapping_defn() self.assertEqual(set(expected_shard_mapping_list1), set(obtained_shard_mapping_list[0])) def test_shard_mapping_list_mappings(self): """Test the listing of all HASH shards in a shard mapping. """ shard_mappings = ShardMapping.list("HASH") self.assertTrue(ShardingUtils.compare_shard_mapping (self.__shard_mapping_1, shard_mappings[0])) def test_fetch_sharding_scheme(self): """Test the fetch method of the HASH sharding scheme. """ hash_sharding_specification_1 = HashShardingSpecification.fetch(1) hash_sharding_specification_2 = HashShardingSpecification.fetch(2) hash_sharding_specification_3 = HashShardingSpecification.fetch(3) hash_sharding_specification_4 = HashShardingSpecification.fetch(4) hash_sharding_specification_5 = HashShardingSpecification.fetch(5) hash_sharding_specifications = HashShardingSpecification.list(1) #list does not return the hashing specifications in order of shard_id, #hence a direct comparison is not posssible. self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_1, hash_sharding_specifications ) ) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_2, hash_sharding_specifications ) ) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_3, hash_sharding_specifications ) ) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_4, hash_sharding_specifications ) ) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_5, hash_sharding_specifications ) ) def test_prune_shard(self): rows = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 500) rows = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 500) rows = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 500) rows = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 500) rows = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 500) status = self.proxy.sharding.prune_shard("db1.t1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables).") rows = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) cnt1 = int(rows[0][0]) self.assertTrue(int(rows[0][0]) < 500) rows = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) cnt2 = int(rows[0][0]) self.assertTrue(int(rows[0][0]) < 500) rows = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) cnt3 = int(rows[0][0]) self.assertTrue(int(rows[0][0]) < 500) rows = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) cnt4 = int(rows[0][0]) self.assertTrue(int(rows[0][0]) < 500) rows = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True}) cnt5 = int(rows[0][0]) self.assertTrue(int(rows[0][0]) < 500) self.assertTrue((cnt1 + cnt2 + cnt3 + cnt4 + cnt5) == 500) def test_prune_lookup(self): self.proxy.sharding.prune_shard("db1.t1") rows = self.__server_2.exec_stmt( "SELECT userID FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_1 = HashShardingSpecification.lookup( val, self.__shard_mapping_id_1, "HASH" ) self.assertEqual( hash_sharding_spec_1.shard_id, 1 ) rows = self.__server_3.exec_stmt( "SELECT userID FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_2 = HashShardingSpecification.lookup( val, self.__shard_mapping_id_1, "HASH" ) self.assertEqual( hash_sharding_spec_2.shard_id, 2 ) rows = self.__server_4.exec_stmt( "SELECT userID FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_3 = HashShardingSpecification.lookup( val, self.__shard_mapping_id_1, "HASH" ) self.assertEqual( hash_sharding_spec_3.shard_id, 3 ) rows = self.__server_5.exec_stmt( "SELECT userID FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_4 = HashShardingSpecification.lookup( val, self.__shard_mapping_id_1, "HASH" ) self.assertEqual( hash_sharding_spec_4.shard_id, 4 ) rows = self.__server_6.exec_stmt( "SELECT userID FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_5 = HashShardingSpecification.lookup( val, self.__shard_mapping_id_1, "HASH" ) self.assertEqual( hash_sharding_spec_5.shard_id, 5 ) def hash_sharding_specification_in_list(self, hash_sharding_spec, hash_sharding_specification_list ): """Verify if the given hash sharding specification is present in the list of hash sharding specifications. :param hash_sharding_spec: The hash sharding specification that needs to be lookedup. :param hash_sharding_specification_list: The list of hash sharding specifications :return: True if the hash sharding specification is present. False otherwise. """ for i in range(0, len(hash_sharding_specification_list)): if ShardingUtils.compare_hash_specifications( hash_sharding_spec, hash_sharding_specification_list[i] ): return True return False def tearDown(self): """Tear down the state store setup. """ self.__server_2.exec_stmt("DROP TABLE db1.t1") self.__server_2.exec_stmt("DROP DATABASE db1") self.__server_3.exec_stmt("DROP TABLE db1.t1") self.__server_3.exec_stmt("DROP DATABASE db1") self.__server_4.exec_stmt("DROP TABLE db1.t1") self.__server_4.exec_stmt("DROP DATABASE db1") self.__server_5.exec_stmt("DROP TABLE db1.t1") self.__server_5.exec_stmt("DROP DATABASE db1") self.__server_6.exec_stmt("DROP TABLE db1.t1") self.__server_6.exec_stmt("DROP DATABASE db1") tests.utils.cleanup_environment() tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestServerClone(tests.utils.TestCase): """Tests the mysqlfabric clone command to clone all the data in a server registered in Fabric into another server. Create a Group and add some servers. Insert data into one of the servers. Now clone this server into the other servers in the group. Now start replication and ensure everything works fine. """ def setUp(self): """Create the setup for performing the testing of the server clone command. """ _LOGGER.critical("\n\nStart of test-fixture-SetUp(): %s\n" % (self._testMethodName)) from __main__ import mysqldump_path, mysqlclient_path self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(0), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__server_1.connect() self.__options_2 = { "uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(1), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__options_3 = { "uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(2), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add(self.__server_3) self.__server_3.connect() #Insert data into Server 1 self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_1.exec_stmt("CREATE DATABASE db1") self.__server_1.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") self.__server_1.exec_stmt("INSERT INTO db1.t1 " "VALUES(101, 'TEST 1')") self.__server_1.exec_stmt("INSERT INTO db1.t1 " "VALUES(202, 'TEST 2')") self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_1.exec_stmt("CREATE DATABASE db2") self.__server_1.exec_stmt("CREATE TABLE db2.t1" "(userID INT, name VARCHAR(30))") self.__server_1.exec_stmt("INSERT INTO db2.t1 " "VALUES(101, 'TEST 1')") self.__server_1.exec_stmt("INSERT INTO db2.t1 " "VALUES(202, 'TEST 2')") _LOGGER.debug("End of test-fixture-SetUp(): %s\n" % (self._testMethodName)) def test_clone_from_group(self): """Verify the clone operation from a group. """ self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) self.__group_1.add_server(self.__server_2) self.__group_1.master = self.__server_2.uuid try: status = self.proxy.server.clone("GROUPID1", self.__server_2.address, str(self.__server_1.uuid)) raise Exception("Cloning to a server inside Fabric should " "throw a fault") except: pass try: status = self.proxy.server.clone("GROUPID1", self.__server_2.address, uuid_server3) raise Exception("Cloning from a server outside the " "source group should throw a fault") except: pass status = self.proxy.server.clone("GROUPID1", self.__server_3.address, None) self.check_xmlrpc_command_result(status) rows = self.__server_3.exec_stmt("SELECT NAME FROM db1.t1", {"fetch": True}) self.assertEqual(len(rows), 2) self.assertEqual(rows[0][0], 'TEST 1') self.assertEqual(rows[1][0], 'TEST 2') rows = self.__server_3.exec_stmt("SELECT NAME FROM db2.t1", {"fetch": True}) self.assertEqual(len(rows), 2) self.assertEqual(rows[0][0], 'TEST 1') self.assertEqual(rows[1][0], 'TEST 2') def test_clone_permissions(self): """Verify the permissions, needed for a clone operation. Change password of the backup user, clone fails on backup. Change password back, Revoke SELECT from the backup user, clone fails on backup. Grant SELECT back, Change password of the restore user, clone fails on restore. Change password back, Revoke CREATE from the restore user, clone fails on restore. Grant CREATE back. Clone succeeds. """ # # Skip test in trial-mode. Users are the same as admin user. # Admin won't be able to grant a privilege back to itself. # if ((MySQLInstances().backup_user == MySQLInstances().user) or (MySQLInstances().restore_user == MySQLInstances().user)): # The trailing comma prevents a newline. print "Skipping test_clone_permissions in trial-mode --- ", return server1 = self.__server_1 server2 = self.__server_2 server3 = self.__server_3 # # Construct a group in the backing store. # self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(server1) self.__group_1.add_server(server2) self.__group_1.master = server2.uuid # # Change password of the the backup user, try a clone, which # fails, because the Fabric instance does still use the old # password, change password back. This works, because Fabric # does not pool the connections for backup and restore users. So # they have to be freshly connected for backup and restore # operations. # server1.exec_stmt( "SET PASSWORD FOR '{user}'@'%' =" " PASSWORD('foobar')".format(user=MySQLInstances().backup_user)) status = self.proxy.server.clone("GROUPID1", server3.address, None) self.check_xmlrpc_command_result(status, has_error=True) server1.exec_stmt("SET PASSWORD FOR '{user}'@'%' =" " PASSWORD('{passwd}')".format( user=MySQLInstances().backup_user, passwd=MySQLInstances().backup_passwd)) # # Revoke the SELECT privilege from the backup user, # try a clone, which fails, grant SELECT back. # server1.exec_stmt("REVOKE SELECT ON *.* FROM '{user}'@'%'".format( user=MySQLInstances().backup_user)) status = self.proxy.server.clone("GROUPID1", server3.address, None) self.check_xmlrpc_command_result(status, has_error=True) server1.exec_stmt("GRANT SELECT ON *.* TO '{user}'@'%'".format( user=MySQLInstances().backup_user)) # # Change password of the the restore user, try a clone, which # fails, because the Fabric instance does still use the old # password, change password back. This works, because Fabric # does not pool the connections for backup and restore users. So # they have to be freshly connected for backup and restore # operations. # server3.exec_stmt( "SET PASSWORD FOR '{user}'@'%' =" " PASSWORD('foobar')".format(user=MySQLInstances().restore_user)) status = self.proxy.server.clone("GROUPID1", server3.address, None) self.check_xmlrpc_command_result(status, has_error=True) server3.exec_stmt("SET PASSWORD FOR '{user}'@'%' =" " PASSWORD('{passwd}')".format( user=MySQLInstances().restore_user, passwd=MySQLInstances().restore_passwd)) # # Revoke the CREATE privilege from the restore user, # try a clone, which fails, grant CREATE back. # server3.exec_stmt("REVOKE CREATE ON *.* FROM '{user}'@'%'".format( user=MySQLInstances().restore_user)) # We have to clear GTID_EXECUTED before we can restore. reset_master(server3) status = self.proxy.server.clone("GROUPID1", server3.address, None) self.check_xmlrpc_command_result(status, has_error=True) server3.exec_stmt("GRANT CREATE ON *.* TO '{user}'@'%'".format( user=MySQLInstances().restore_user)) # We have to clear GTID_EXECUTED before we can try the next restore. reset_master(server3) # # Run a successful clone. # status = self.proxy.server.clone("GROUPID1", server3.address, None) self.check_xmlrpc_command_result(status) rows = server3.exec_stmt("SELECT NAME FROM db1.t1", {"fetch": True}) self.assertEqual(len(rows), 2) self.assertEqual(rows[0][0], 'TEST 1') self.assertEqual(rows[1][0], 'TEST 2') rows = server3.exec_stmt("SELECT NAME FROM db2.t1", {"fetch": True}) self.assertEqual(len(rows), 2) self.assertEqual(rows[0][0], 'TEST 1') self.assertEqual(rows[1][0], 'TEST 2') def test_clone_error(self): """Verify that the clone operations throws an error for wrong input values. """ try: status = self.proxy.server.clone(None, self.__server_3.address, str(self.__server_1.uuid)) raise Exception("Not providing a group "\ "should throw a fault") except: pass def tearDown(self): """Clean up the existing environment """ _LOGGER.debug("Start of test-fixture-tearDown(): %s" % (self._testMethodName)) tests.utils.cleanup_environment() _LOGGER.debug("End of test-fixture-tearDown(): %s" % (self._testMethodName))
class TestShardingServices(tests.utils.TestCase): def setUp(self): """Configure the existing environment """ self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID1 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_2.exec_stmt("CREATE DATABASE db2") self.__server_2.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(201, 401): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_2.exec_stmt("CREATE DATABASE db3") self.__server_2.exec_stmt("CREATE TABLE db3.t3" "(userID3 INT, name VARCHAR(30))") for i in range(401, 601): self.__server_2.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add( self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID1 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_3.exec_stmt("CREATE DATABASE db2") self.__server_3.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(201, 401): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_3.exec_stmt("CREATE DATABASE db3") self.__server_3.exec_stmt("CREATE TABLE db3.t3" "(userID3 INT, name VARCHAR(30))") for i in range(401, 601): self.__server_3.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add( self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(3), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID1 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_4.exec_stmt("CREATE DATABASE db2") self.__server_4.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(201, 401): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_4.exec_stmt("CREATE DATABASE db3") self.__server_4.exec_stmt("CREATE TABLE db3.t3" "(userID3 INT, name VARCHAR(30))") for i in range(401, 601): self.__server_4.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add( self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(4), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID1 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_5.exec_stmt("CREATE DATABASE db2") self.__server_5.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(201, 401): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_5.exec_stmt("CREATE DATABASE db3") self.__server_5.exec_stmt("CREATE TABLE db3.t3" "(userID3 INT, name VARCHAR(30))") for i in range(401, 601): self.__server_5.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add( self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(5), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__server_6.connect() self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_6.exec_stmt("CREATE DATABASE db1") self.__server_6.exec_stmt("CREATE TABLE db1.t1" "(userID1 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_6.exec_stmt("CREATE DATABASE db2") self.__server_6.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(201, 401): self.__server_6.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_6.exec_stmt("CREATE DATABASE db3") self.__server_6.exec_stmt("CREATE TABLE db3.t3" "(userID3 INT, name VARCHAR(30))") for i in range(401, 601): self.__server_6.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add( self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) status = self.proxy.sharding.create_definition("HASH", "GROUPID1") self.check_xmlrpc_command_result(status, returns=1) status = self.proxy.sharding.add_table(1, "db1.t1", "userID1") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_table(1, "db2.t2", "userID2") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_table(1, "db3.t3", "userID3") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_shard( 1, "GROUPID2,GROUPID3,GROUPID4,GROUPID5,GROUPID6", "ENABLED" ) self.check_xmlrpc_command_result(status) status = self.proxy.sharding.prune_shard("db1.t1") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.prune_shard("db2.t2") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.prune_shard("db3.t3") self.check_xmlrpc_command_result(status) def tearDown(self): """Clean up the existing environment """ tests.utils.cleanup_environment() def test_prune_lookup_shard1(self): '''Verify that after the prune the lookup of any pruned value in the shard results in looking up the same shard. ''' status = self.proxy.sharding.prune_shard("db1.t1") self.check_xmlrpc_command_result(status) rows = self.__server_2.exec_stmt("SELECT userID1 FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_1.shard_id, 1 ) rows = self.__server_3.exec_stmt( "SELECT userID1 FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_2.shard_id, 2 ) rows = self.__server_4.exec_stmt( "SELECT userID1 FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_3.shard_id, 3 ) rows = self.__server_5.exec_stmt( "SELECT userID1 FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_4 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_4.shard_id, 4 ) rows = self.__server_6.exec_stmt( "SELECT userID1 FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_5.shard_id, 5 ) def test_prune_lookup_shard2(self): '''Verify that after the prune the lookup of any pruned value in the shard results in looking up the same shard. ''' status = self.proxy.sharding.prune_shard("db2.t2") self.check_xmlrpc_command_result(status) rows = self.__server_2.exec_stmt("SELECT userID2 FROM db2.t2", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_1.shard_id, 1 ) rows = self.__server_3.exec_stmt( "SELECT userID2 FROM db2.t2", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_2.shard_id, 2 ) rows = self.__server_4.exec_stmt( "SELECT userID2 FROM db2.t2", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_3.shard_id, 3 ) rows = self.__server_5.exec_stmt( "SELECT userID2 FROM db2.t2", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_4 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_4.shard_id, 4 ) rows = self.__server_6.exec_stmt( "SELECT userID2 FROM db2.t2", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_5.shard_id, 5 ) def test_prune_lookup_shard3(self): '''Verify that after the prune the lookup of any pruned value in the shard results in looking up the same shard. ''' status = self.proxy.sharding.prune_shard("db3.t3") self.check_xmlrpc_command_result(status) rows = self.__server_2.exec_stmt("SELECT userID3 FROM db3.t3", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_1.shard_id, 1 ) rows = self.__server_3.exec_stmt( "SELECT userID3 FROM db3.t3", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_2.shard_id, 2 ) rows = self.__server_4.exec_stmt( "SELECT userID3 FROM db3.t3", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_3.shard_id, 3 ) rows = self.__server_5.exec_stmt( "SELECT userID3 FROM db3.t3", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_4 = HashShardingSpecification.lookup( val, 1, "HASH" ) self.assertEqual( hash_sharding_spec_4.shard_id, 4 ) rows = self.__server_6.exec_stmt( "SELECT userID3 FROM db3.t3", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_5.shard_id, 5 ) def test_list_shard_mappings(self): expected_shard_mapping = { 'mapping_id' : 1, 'type_name' : 'HASH', 'global_group_id' : 'GROUPID1' } status = self.proxy.sharding.list_definitions() obtained_shard_mapping = self.check_xmlrpc_simple(status, {}) self.assertEqual(expected_shard_mapping, obtained_shard_mapping) def test_lookup_shard_mapping(self): expected_shard_mapping_1 = { 'mapping_id' : 1, 'table_name' : 'db1.t1', 'column_name' : 'userID1', 'type_name' : 'HASH', 'global_group' : 'GROUPID1' } status = self.proxy.sharding.lookup_table("db1.t1") obtained_shard_mapping = self.check_xmlrpc_simple(status, {}) self.assertEqual(expected_shard_mapping_1, obtained_shard_mapping) expected_shard_mapping_2 = { 'mapping_id' : 1, 'table_name' : 'db2.t2', 'column_name' : 'userID2', 'type_name' : 'HASH', 'global_group' : 'GROUPID1' } status = self.proxy.sharding.lookup_table("db2.t2") obtained_shard_mapping = self.check_xmlrpc_simple(status, {}) self.assertEqual(expected_shard_mapping_2, obtained_shard_mapping)
class TestShardingPrune(tests.utils.TestCase): """Contains unit tests for testing the shard split operation and for verifying that the global server configuration remains constant after the shard split configuration. """ def setUp(self): """Creates the topology for testing. """ tests.utils.cleanup_environment() self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__server_1.connect() self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID VARCHAR(20) PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("a"+str(i), i)) for i in range(101, 401): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("c"+str(i), i)) for i in range(10001, 10601): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("d"+str(i), i)) for i in range(100001, 100801): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("e"+str(i), i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_2.exec_stmt("CREATE DATABASE db2") self.__server_2.exec_stmt("CREATE TABLE db2.t2" "(userID VARCHAR(20), salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("a"+str(i), i)) for i in range(101, 401): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("c"+str(i), i)) for i in range(10001, 10601): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("d"+str(i), i)) for i in range(100001, 100801): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("e"+str(i), i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add( self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("a"+str(i), i)) for i in range(101, 401): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("c"+str(i), i)) for i in range(10001, 10601): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("d"+str(i), i)) for i in range(100001, 100801): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("e"+str(i), i)) self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_3.exec_stmt("CREATE DATABASE db2") self.__server_3.exec_stmt("CREATE TABLE db2.t2" "(userID VARCHAR(20), salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("a"+str(i), i)) for i in range(101, 401): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("c"+str(i), i)) for i in range(10001, 10601): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("d"+str(i), i)) for i in range(100001, 100801): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("e"+str(i), i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add( self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(3), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("a"+str(i), i)) for i in range(101, 401): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("c"+str(i), i)) for i in range(10001, 10601): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("d"+str(i), i)) for i in range(100001, 100801): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("e"+str(i), i)) self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_4.exec_stmt("CREATE DATABASE db2") self.__server_4.exec_stmt("CREATE TABLE db2.t2" "(userID VARCHAR(20), salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("a"+str(i), i)) for i in range(101, 401): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("c"+str(i), i)) for i in range(10001, 10601): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("d"+str(i), i)) for i in range(100001, 100801): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("e"+str(i), i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add( self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(4), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("a"+str(i), i)) for i in range(101, 401): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("c"+str(i), i)) for i in range(10001, 10601): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("d"+str(i), i)) for i in range(100001, 100801): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("e"+str(i), i)) self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_5.exec_stmt("CREATE DATABASE db2") self.__server_5.exec_stmt("CREATE TABLE db2.t2" "(userID VARCHAR(20), salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("a"+str(i), i)) for i in range(101, 401): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("c"+str(i), i)) for i in range(10001, 10601): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("d"+str(i), i)) for i in range(100001, 100801): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("e"+str(i), i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add( self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(5), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__server_6.connect() self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_6.exec_stmt("CREATE DATABASE db1") self.__server_6.exec_stmt("CREATE TABLE db1.t1" "(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("a"+str(i), i)) for i in range(101, 401): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("c"+str(i), i)) for i in range(10001, 10601): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("d"+str(i), i)) for i in range(100001, 100801): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("e"+str(i), i)) self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_6.exec_stmt("CREATE DATABASE db2") self.__server_6.exec_stmt("CREATE TABLE db2.t2" "(userID VARCHAR(20), salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71): self.__server_6.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("a"+str(i), i)) for i in range(101, 401): self.__server_6.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_6.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("c"+str(i), i)) for i in range(10001, 10601): self.__server_6.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("d"+str(i), i)) for i in range(100001, 100801): self.__server_6.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("e"+str(i), i)) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add( self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) status = self.proxy.sharding.create_definition("RANGE_STRING", "GROUPID1") self.check_xmlrpc_command_result(status, returns=1) status = self.proxy.sharding.add_table(1, "db1.t1", "userID") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_table(1, "db2.t2", "userID") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_shard( 1, "GROUPID2/a,GROUPID3/b,GROUPID4/c,GROUPID5/d,GROUPID6/e", "ENABLED" ) self.check_xmlrpc_command_result(status) def test_prune_shard(self): status = self.proxy.sharding.prune_shard("db1.t1") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.lookup_servers("db1.t1", "a3", "LOCAL") for info in self.check_xmlrpc_iter(status): if info['status'] == MySQLServer.PRIMARY: shard_server = fetch_test_server(info['server_uuid']) shard_server.connect() rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) self.assertTrue(int(rows[0][0]) == 70) status = self.proxy.sharding.prune_shard("db1.t1") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.lookup_servers("db1.t1", "b12", "LOCAL") for info in self.check_xmlrpc_iter(status): if info['status'] == MySQLServer.PRIMARY: shard_server = fetch_test_server(info['server_uuid']) shard_server.connect() rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) self.assertTrue(int(rows[0][0]) == 300) rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) self.assertTrue(int(rows[0][0]) == 300) status = self.proxy.sharding.prune_shard("db1.t1") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.lookup_servers("db1.t1", "c35", "LOCAL") for info in self.check_xmlrpc_iter(status): if info['status'] == MySQLServer.PRIMARY: shard_server = fetch_test_server(info['server_uuid']) shard_server.connect() rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) self.assertTrue(int(rows[0][0]) == 200) rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) self.assertTrue(int(rows[0][0]) == 200) status = self.proxy.sharding.prune_shard("db1.t1") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.lookup_servers("db1.t1", "d21", "LOCAL") for info in self.check_xmlrpc_iter(status): if info['status'] == MySQLServer.PRIMARY: shard_server = fetch_test_server(info['server_uuid']) shard_server.connect() rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) self.assertTrue(int(rows[0][0]) == 600) rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) self.assertTrue(int(rows[0][0]) == 600) status = self.proxy.sharding.prune_shard("db1.t1") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.lookup_servers("db1.t1", "e31", "LOCAL") for info in self.check_xmlrpc_iter(status): if info['status'] == MySQLServer.PRIMARY: shard_server = fetch_test_server(info['server_uuid']) shard_server.connect() rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) self.assertTrue(int(rows[0][0]) == 800) rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) self.assertTrue(int(rows[0][0]) == 800) def tearDown(self): """Clean up the existing environment """ tests.utils.cleanup_environment()
def test_privileges(self): """Test whether user's have the appropriate privileges. """ # Some privileges MINIMUM_PRIVILEGES = [ "REPLICATION SLAVE", "REPLICATION CLIENT", "SUPER", "SHOW DATABASES", "RELOAD" ] # Connect to server as root and create temporary user. uuid = MySQLServer.discover_uuid(OPTIONS["address"]) server = MySQLServer( _uuid.UUID(uuid), OPTIONS["address"], tests.utils.MySQLInstances().root_user, tests.utils.MySQLInstances().root_passwd ) ConnectionPool().purge_connections(_uuid.UUID(uuid)) server.connect() server.set_session_binlog(False) server.exec_stmt( "CREATE USER 'jeffrey'@'%%' IDENTIFIED BY 'mypass'" ) # Check if jeffrey (temporary user) has the appropriate privileges. # There is not privilege associate to jeffrey. new_server = MySQLServer( _uuid.UUID(uuid), OPTIONS["address"], "jeffrey", "mypass" ) new_server.connect() self.assertFalse( new_server.has_privileges(MINIMUM_PRIVILEGES) ) # Check if jeffrey (temporary user) has the appropriate privileges. # Grant required privileges except RELOAD # There is no RELOAD on a global level. privileges=", ".join([priv for priv in MINIMUM_PRIVILEGES if priv != "RELOAD"] ) server.exec_stmt( "GRANT {privileges} ON *.* TO 'jeffrey'@'%%'".format( privileges=privileges) ) server.exec_stmt("FLUSH PRIVILEGES") self.assertFalse( new_server.has_privileges(MINIMUM_PRIVILEGES) ) # Check if jeffrey (temporary user) has the appropriate privileges. # The RELOAD on a global level was granted. server.exec_stmt("GRANT RELOAD ON *.* TO 'jeffrey'@'%%'") server.exec_stmt("FLUSH PRIVILEGES") self.assertTrue( new_server.has_privileges(MINIMUM_PRIVILEGES) ) # Check if jeffrey (temporary user) has the appropriate privileges. # Revoke privilegs from temporary user. # There is no ALL on a global level. server.exec_stmt("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " "'jeffrey'@'%%'" ) server.exec_stmt("GRANT ALL ON fabric.* TO 'jeffrey'@'%%'") server.exec_stmt("FLUSH PRIVILEGES") self.assertFalse( new_server.has_privileges(MINIMUM_PRIVILEGES) ) # Check if jeffrey (temporary user) has the appropriate privileges. # The ALL on a global level was granted. server.exec_stmt("GRANT ALL ON *.* TO 'jeffrey'@'%%'") server.exec_stmt("FLUSH PRIVILEGES") self.assertTrue( new_server.has_privileges(MINIMUM_PRIVILEGES) ) # Drop temporary user. server.exec_stmt("DROP USER 'jeffrey'@'%%'") server.set_session_binlog(True) server.disconnect() new_server.disconnect() ConnectionPool().purge_connections(_uuid.UUID(uuid))
class TestShardingPrune(unittest.TestCase): """Contains unit tests for testing the shard split operation and for verifying that the global server configuration remains constant after the shard split configuration. """ def assertStatus(self, status, expect): items = (item['diagnosis'] for item in status[1] if item['diagnosis']) self.assertEqual(status[1][-1]["success"], expect, "\n".join(items)) def setUp(self): """Creates the topology for testing. """ tests.utils.cleanup_environment() self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__server_1.connect() self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(101, 301): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(1001, 1201): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(10001, 10201): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(100001, 100201): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_2.exec_stmt("CREATE DATABASE db2") self.__server_2.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(101, 301): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(1001, 1201): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(10001, 10201): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(100001, 100201): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add( self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(101, 301): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(1001, 1201): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(10001, 10201): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(100001, 100201): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_3.exec_stmt("CREATE DATABASE db2") self.__server_3.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(101, 301): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(1001, 1201): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(10001, 10201): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(100001, 100201): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add( self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(3), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(101, 301): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(1001, 1201): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(10001, 10201): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(100001, 100201): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_4.exec_stmt("CREATE DATABASE db2") self.__server_4.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(101, 301): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(1001, 1201): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(10001, 10201): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(100001, 100201): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add( self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(4), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(101, 301): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(1001, 1201): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(10001, 10201): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(100001, 100201): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_5.exec_stmt("CREATE DATABASE db2") self.__server_5.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(101, 301): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(1001, 1201): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(10001, 10201): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(100001, 100201): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add( self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(5), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__server_6.connect() self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add( self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) status = self.proxy.sharding.create_definition("RANGE", "GROUPID1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_define_shard_mapping).") self.assertEqual(status[2], 1) status = self.proxy.sharding.add_table(1, "db1.t1", "userID") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_table(1, "db2.t2", "userID") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_shard( 1, "GROUPID2/1,GROUPID3/101,GROUPID4/1001,GROUPID5/10001", "ENABLED" ) self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard).") status = self.proxy.sharding.prune_shard("db1.t1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables).") def test_split_shard_1(self): '''Test the split of shard 1 and the global server configuration after that. The test splits shard 1 between GROUPID2 and GROUPID6. After the split is done, it verifies the count on GROUPID6 to check that the group has tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. ''' row_cnt_shard_before_split_db1_t1 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_before_split_db2_t2 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_before_split_db1_t1 = \ int(row_cnt_shard_before_split_db1_t1[0][0]) row_cnt_shard_before_split_db2_t2 = \ int(row_cnt_shard_before_split_db2_t2[0][0]) self.assertEqual(row_cnt_shard_before_split_db1_t1, 70) self.assertEqual(row_cnt_shard_before_split_db2_t2, 70) status = self.proxy.sharding.split_shard("1", "GROUPID6", "36") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables_after_split).") row_cnt_shard_after_split_db1_t1_a = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_db1_t1_a = \ int(row_cnt_shard_after_split_db1_t1_a[0][0]) self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 35) row_cnt_shard_after_split_db2_t2_a = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_db2_t2_a = \ int(row_cnt_shard_after_split_db2_t2_a[0][0]) self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 35) row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_db1_t1_b = \ int(row_cnt_shard_after_split_db1_t1_b[0][0]) self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 35) row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_db2_t2_b = \ int(row_cnt_shard_after_split_db2_t2_b[0][0]) self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 35) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_split_shard_2(self): '''Test the split of shard 2 and the global server configuration after that. The test splits shard 2 between GROUPID3 and GROUPID6. After the split is done, it verifies the count on GROUPID6 to check that the group has tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. ''' row_cnt_shard_before_split_db1_t1 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_before_split_db2_t2 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_before_split_db1_t1 = \ int(row_cnt_shard_before_split_db1_t1[0][0]) row_cnt_shard_before_split_db2_t2 = \ int(row_cnt_shard_before_split_db2_t2[0][0]) self.assertEqual(row_cnt_shard_before_split_db1_t1, 200) self.assertEqual(row_cnt_shard_before_split_db2_t2, 200) status = self.proxy.sharding.split_shard("2", "GROUPID6", "201") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables_after_split).") row_cnt_shard_after_split_db1_t1_a = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_db1_t1_a = \ int(row_cnt_shard_after_split_db1_t1_a[0][0]) self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 100) row_cnt_shard_after_split_db2_t2_a = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_db2_t2_a = \ int(row_cnt_shard_after_split_db2_t2_a[0][0]) self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 100) row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_db1_t1_b = \ int(row_cnt_shard_after_split_db1_t1_b[0][0]) self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 100) row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_db2_t2_b = \ int(row_cnt_shard_after_split_db2_t2_b[0][0]) self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 100) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_split_shard_3(self): '''Test the split of shard 3 and the global server configuration after that. The test splits shard 3 between GROUPID4 and GROUPID6. After the split is done, it verifies the count on GROUPID6 to check that the group has tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. ''' row_cnt_shard_before_split_db1_t1 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_before_split_db2_t2 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_before_split_db1_t1 = \ int(row_cnt_shard_before_split_db1_t1[0][0]) row_cnt_shard_before_split_db2_t2 = \ int(row_cnt_shard_before_split_db2_t2[0][0]) self.assertEqual(row_cnt_shard_before_split_db1_t1, 200) self.assertEqual(row_cnt_shard_before_split_db2_t2, 200) status = self.proxy.sharding.split_shard("3", "GROUPID6", "1101") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables_after_split).") row_cnt_shard_after_split_db1_t1_a = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_db1_t1_a = \ int(row_cnt_shard_after_split_db1_t1_a[0][0]) self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 100) row_cnt_shard_after_split_db2_t2_a = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_db2_t2_a = \ int(row_cnt_shard_after_split_db2_t2_a[0][0]) self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 100) row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_db1_t1_b = \ int(row_cnt_shard_after_split_db1_t1_b[0][0]) self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 100) row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_db2_t2_b = \ int(row_cnt_shard_after_split_db2_t2_b[0][0]) self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 100) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_split_shard_4(self): '''Test the split of shard 4 and the global server configuration after that. The test splits shard 4 between GROUPID5 and GROUPID6. After the split is done, it verifies the count on GROUPID6 to check that the group has tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. ''' row_cnt_shard_before_split_db1_t1 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_before_split_db2_t2 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_before_split_db1_t1 = \ int(row_cnt_shard_before_split_db1_t1[0][0]) row_cnt_shard_before_split_db2_t2 = \ int(row_cnt_shard_before_split_db2_t2[0][0]) self.assertEqual(row_cnt_shard_before_split_db1_t1, 400) self.assertEqual(row_cnt_shard_before_split_db2_t2, 400) status = self.proxy.sharding.split_shard("4", "GROUPID6", "100001") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables_after_split).") row_cnt_shard_after_split_db1_t1_a = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_db1_t1_a = \ int(row_cnt_shard_after_split_db1_t1_a[0][0]) self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 200) row_cnt_shard_after_split_db2_t2_a = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_db2_t2_a = \ int(row_cnt_shard_after_split_db2_t2_a[0][0]) self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 200) row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_db1_t1_b = \ int(row_cnt_shard_after_split_db1_t1_b[0][0]) self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 200) row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_db2_t2_b = \ int(row_cnt_shard_after_split_db2_t2_b[0][0]) self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 200) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_split_shard_error_1(self): '''Test that splitting with an invalid split value fails to split a shard. ''' status = self.proxy.sharding.split_shard("3", "GROUPID6", "10") self.assertStatus(status, _executor.Job.ERROR) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Tried to execute action (_check_shard_information).") def test_split_shard_error_2(self): '''Test that splitting with an invalid split value fails to split a shard. ''' status = self.proxy.sharding.split_shard("3", "GROUPID6", "1101.5") self.assertStatus(status, _executor.Job.ERROR) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Tried to execute action (_check_shard_information).") def test_split_shard_error_3(self): '''Test that splitting with an invalid split value fails to split a shard. ''' self.__group_6.status = Group.INACTIVE status = self.proxy.sharding.split_shard("3", "GROUPID6", "1101.5") self.assertStatus(status, _executor.Job.ERROR) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Tried to execute action (_check_shard_information).") def test_split_shard_error_4(self): '''Test that splitting with an empty split value fails to split a shard. ''' self.__group_6.status = Group.INACTIVE status = self.proxy.sharding.split_shard("3", "GROUPID6", None) self.assertStatus(status, _executor.Job.ERROR) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Tried to execute action (_check_shard_information).") def tearDown(self): status = self.proxy.sharding.disable_shard("1") status = self.proxy.sharding.disable_shard("2") status = self.proxy.sharding.disable_shard("3") status = self.proxy.sharding.disable_shard("4") status = self.proxy.sharding.disable_shard("5") status = self.proxy.sharding.disable_shard("6") status = self.proxy.sharding.remove_shard("1") status = self.proxy.sharding.remove_shard("2") status = self.proxy.sharding.remove_shard("3") status = self.proxy.sharding.remove_shard("4") status = self.proxy.sharding.remove_shard("5") status = self.proxy.sharding.remove_shard("6") status = self.proxy.sharding.remove_table("db1.t1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_remove_shard_mapping).") status = self.proxy.sharding.remove_table("db2.t2") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_remove_shard_mapping).") status = self.proxy.sharding.remove_definition("1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_remove_shard_mapping_defn).") self.proxy.group.demote("GROUPID1") self.proxy.group.demote("GROUPID2") self.proxy.group.demote("GROUPID3") self.proxy.group.demote("GROUPID4") self.proxy.group.demote("GROUPID5") self.proxy.group.demote("GROUPID6") for group_id in ("GROUPID1", "GROUPID2", "GROUPID3", "GROUPID4", "GROUPID5", "GROUPID6"): status = self.proxy.group.lookup_servers(group_id) self.assertEqual(status[0], True) self.assertEqual(status[1], "") obtained_server_list = status[2] status = self.proxy.group.remove( group_id, obtained_server_list[0]["server_uuid"] ) self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_remove_server).") status = self.proxy.group.destroy(group_id) self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_destroy_group).") tests.utils.cleanup_environment() tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestServerClone(unittest.TestCase): """Tests the mysqlfabric clone command to clone all the data in a server registered in Fabric into another server. Create a Group and add some servers. Insert data into one of the servers. Now clone this server into the other servers in the group. Now start replication and ensure everything works fine. """ def assertStatus(self, status, expect): """Asserts that the status obtained is similar to the status expected. """ items = (item['diagnosis'] for item in status[1] if item['diagnosis']) self.assertEqual(status[1][-1]["success"], expect, "\n".join(items)) def setUp(self): """Create the setup for performing the testing of the server clone command. """ from __main__ import mysqldump_path, mysqlclient_path self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__server_1.connect() self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address":MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__options_3 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address":MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add(self.__server_3) self.__server_3.connect() #Insert data into Server 1 self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_1.exec_stmt("CREATE DATABASE db1") self.__server_1.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") self.__server_1.exec_stmt("INSERT INTO db1.t1 " "VALUES(101, 'TEST 1')") self.__server_1.exec_stmt("INSERT INTO db1.t1 " "VALUES(202, 'TEST 2')") self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_1.exec_stmt("CREATE DATABASE db2") self.__server_1.exec_stmt("CREATE TABLE db2.t1" "(userID INT, name VARCHAR(30))") self.__server_1.exec_stmt("INSERT INTO db2.t1 " "VALUES(101, 'TEST 1')") self.__server_1.exec_stmt("INSERT INTO db2.t1 " "VALUES(202, 'TEST 2')") def test_clone_from_group(self): """Verify the clone operation from a group. """ self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) self.__group_1.add_server(self.__server_2) self.__group_1.master = self.__server_2.uuid try: status = self.proxy.server.clone("GROUPID1", self.__server_2.address, str(self.__server_1.uuid)) raise Exception("Cloning to a server inside Fabric should " "throw a fault") except: pass try: status = self.proxy.server.clone("GROUPID1", self.__server_2.address, uuid_server3) raise Exception("Cloning from a server outside the " "source group should throw a fault") except: pass status = self.proxy.server.clone("GROUPID1", self.__server_3.address, None) self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_restore_server).") rows = self.__server_3.exec_stmt( "SELECT NAME FROM db1.t1", {"fetch" : True}) self.assertEqual(len(rows), 2) self.assertEqual(rows[0][0], 'TEST 1') self.assertEqual(rows[1][0], 'TEST 2') rows = self.__server_3.exec_stmt( "SELECT NAME FROM db2.t1", {"fetch" : True}) self.assertEqual(len(rows), 2) self.assertEqual(rows[0][0], 'TEST 1') self.assertEqual(rows[1][0], 'TEST 2') def test_clone_error(self): """Verify that the clone operations throws an error for wrong input values. """ try: status = self.proxy.server.clone(None, self.__server_3.address, str(self.__server_1.uuid)) raise Exception("Not providing a group "\ "should throw a fault") except: pass def tearDown(self): """Clean up the existing environment """ self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_1.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2") tests.utils.cleanup_environment() tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestBackupMySQLDump(unittest.TestCase): """Test taking a backup from a source group to a destination group. The source group handles the source shard and the destination group handles the destination shard. The backup and restore helps take up a backup of the source shard and setup the destination shard. """ def setUp(self): """Configure the existing environment """ from __main__ import mysqldump_path, mysqlclient_path self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__server_1.connect() self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address":MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_1.exec_stmt("DROP DATABASE IF EXISTS backup_db") self.__server_1.exec_stmt("CREATE DATABASE backup_db") self.__server_1.exec_stmt("CREATE TABLE backup_db.backup_table" "(userID INT, name VARCHAR(30))") self.__server_1.exec_stmt("CREATE TABLE backup_db.trigger_table" "(count INT)") self.__server_1.exec_stmt("INSERT INTO backup_db.trigger_table " "VALUES(0)") self.__server_1.exec_stmt("CREATE TRIGGER backup_db.backup_table_ai" " AFTER INSERT ON backup_db.backup_table" " FOR EACH ROW UPDATE backup_db.trigger_table" " SET count = count + 1") self.__server_1.exec_stmt("INSERT INTO backup_db.backup_table " "VALUES(101, 'TEST 1')") self.__server_1.exec_stmt("INSERT INTO backup_db.backup_table " "VALUES(202, 'TEST 2')") self.__server_1.exec_stmt("CREATE INDEX i1 ON" " backup_db.backup_table (userID)") self.__server_1.exec_stmt("CREATE EVENT backup_db.ev1" " ON SCHEDULE EVERY 1 DAY DO SELECT 1") self.mysqldump_path = mysqldump_path self.mysqlclient_path = mysqlclient_path def test_backup(self): image = MySQLDump.backup(self.__server_1, MySQLInstances().backup_user, MySQLInstances().backup_passwd, self.mysqldump_path) MySQLDump.restore_fabric_server(self.__server_2, MySQLInstances().restore_user, MySQLInstances().restore_passwd, image, self.mysqlclient_path) rows = self.__server_2.exec_stmt( "SELECT NAME FROM backup_db.backup_table", {"fetch" : True}) self.assertEqual(len(rows), 2) self.assertEqual(rows[0][0], 'TEST 1') self.assertEqual(rows[1][0], 'TEST 2') def tearDown(self): """Clean up the existing environment """ tests.utils.cleanup_environment()
class TestHashMoveGlobal(tests.utils.TestCase): """Contains unit tests for testing the shard move operation and for verifying that the global server configuration remains constant after the shard move configuration. """ def setUp(self): """Configure the existing environment """ tests.utils.cleanup_environment() self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__server_1.connect() self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_2.exec_stmt("CREATE DATABASE db2") self.__server_2.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add( self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_3.exec_stmt("CREATE DATABASE db2") self.__server_3.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add( self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(3), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_4.exec_stmt("CREATE DATABASE db2") self.__server_4.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add( self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(4), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_5.exec_stmt("CREATE DATABASE db2") self.__server_5.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add( self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(5), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add( self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) status = self.proxy.sharding.create_definition("HASH", "GROUPID1") self.check_xmlrpc_command_result(status, returns=1) status = self.proxy.sharding.add_table(1, "db1.t1", "userID") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_table(1, "db2.t2", "userID") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_shard( 1, "GROUPID2,GROUPID3,GROUPID4,GROUPID5", "ENABLED" ) self.check_xmlrpc_command_result(status) status = self.proxy.sharding.prune_shard("db1.t1") self.check_xmlrpc_command_result(status) def test_move_shard_1(self): '''Test the move of shard 1 and the global server configuration after that. The test moves shard 1 from GROUPID2 to GROUPID6. After the move is done, it verifies the count on GROUPID6 to check that the group has all the tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. GROUPID2 should not have received the values since it has had the shard moved away from it. ''' row_cnt_shard_before_move_db1_t1 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_before_move_db2_t2 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_before_move_db1_t1 = \ int(row_cnt_shard_before_move_db1_t1[0][0]) row_cnt_shard_before_move_db2_t2 = \ int(row_cnt_shard_before_move_db2_t2[0][0]) status = self.proxy.sharding.move_shard("1", "GROUPID6") self.check_xmlrpc_command_result(status) self.__server_6.connect() row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_move_db1_t1 = \ int(row_cnt_shard_after_move_db1_t1[0][0]) row_cnt_shard_after_move_db2_t2 = \ int(row_cnt_shard_after_move_db2_t2[0][0]) self.assertTrue( row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1 ) self.assertTrue( row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2 ) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) try: #Verify that the data is not there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) raise Exception("Server should not be connected to global server") except _errors.DatabaseError: pass #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_move_shard_2(self): '''Test the move of shard 2 and the global server configuration after that. The test moves shard 2 from GROUPID3 to GROUPID6. After the move is done, it verifies the count on GROUPID6 to check that the group has all the tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. GROUPID3 should not have received the values since it has had the shard moved away from it. ''' row_cnt_shard_before_move_db1_t1 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_before_move_db2_t2 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_before_move_db1_t1 = \ int(row_cnt_shard_before_move_db1_t1[0][0]) row_cnt_shard_before_move_db2_t2 = \ int(row_cnt_shard_before_move_db2_t2[0][0]) status = self.proxy.sharding.move_shard("2", "GROUPID6") self.check_xmlrpc_command_result(status) self.__server_6.connect() row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_move_db1_t1 = \ int(row_cnt_shard_after_move_db1_t1[0][0]) row_cnt_shard_after_move_db2_t2 = \ int(row_cnt_shard_after_move_db2_t2[0][0]) self.assertTrue( row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1 ) self.assertTrue( row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2 ) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) try: #Verify that the data is not there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) raise Exception("Server should not be connected to global server") except _errors.DatabaseError: pass #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_move_shard_3(self): '''Test the move of shard 3 and the global server configuration after that. The test moves shard 3 from GROUPID4 to GROUPID6. After the move is done, it verifies the count on GROUPID6 to check that the group has all the tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. GROUPID4 should not have received the values since it has had the shard moved away from it. ''' row_cnt_shard_before_move_db1_t1 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_before_move_db2_t2 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_before_move_db1_t1 = \ int(row_cnt_shard_before_move_db1_t1[0][0]) row_cnt_shard_before_move_db2_t2 = \ int(row_cnt_shard_before_move_db2_t2[0][0]) status = self.proxy.sharding.move_shard("3", "GROUPID6") self.check_xmlrpc_command_result(status) self.__server_6.connect() row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_move_db1_t1 = \ int(row_cnt_shard_after_move_db1_t1[0][0]) row_cnt_shard_after_move_db2_t2 = \ int(row_cnt_shard_after_move_db2_t2[0][0]) self.assertTrue( row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1 ) self.assertTrue( row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2 ) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) try: #Verify that the data is not there in the third shard's #original group. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) raise Exception("Server should not be connected to global server") except _errors.DatabaseError: pass #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard #new group. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_move_shard_4(self): '''Test the move of shard 4 and the global server configuration after that. The test moves shard 4 from GROUPID5 to GROUPID6. After the move is done, it verifies the count on GROUPID6 to check that the group has all the tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. GROUPID5 should not have received the values since it has had the shard moved away from it. ''' row_cnt_shard_before_move_db1_t1 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_before_move_db2_t2 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_before_move_db1_t1 = \ int(row_cnt_shard_before_move_db1_t1[0][0]) row_cnt_shard_before_move_db2_t2 = \ int(row_cnt_shard_before_move_db2_t2[0][0]) status = self.proxy.sharding.move_shard("4", "GROUPID6") self.check_xmlrpc_command_result(status) self.__server_6.connect() row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_move_db1_t1 = \ int(row_cnt_shard_after_move_db1_t1[0][0]) row_cnt_shard_after_move_db2_t2 = \ int(row_cnt_shard_after_move_db2_t2[0][0]) self.assertTrue( row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1 ) self.assertTrue( row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2 ) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) try: #Verify that the data is not there in the fourth shard's #original group. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) raise Exception("Server should not be connected to global server") except _errors.DatabaseError: pass #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard's new #group. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def tearDown(self): """Clean up the existing environment """ tests.utils.cleanup_environment()
class TestSharding(unittest.TestCase): def setUp(self): """Configure the existing environment """ self.__options_1 = { "uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address": "server_1.mysql.com:3060", "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__options_2 = { "uuid": _uuid.UUID("{aa75a12a-98d1-414c-96af-9e9d4b179678}"), "address": "server_2.mysql.com:3060", "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) self.__group_1.add_server(self.__server_2) self.__options_3 = { "uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(0), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add(self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS prune_db") self.__server_3.exec_stmt("CREATE DATABASE prune_db") self.__server_3.exec_stmt("CREATE TABLE prune_db.prune_table" "(userID INT, name VARCHAR(30))") self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table " "VALUES(101, 'TEST 1')") self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table " "VALUES(202, 'TEST 2')") self.__options_4 = { "uuid": _uuid.UUID("{dd75a12a-98d1-414c-96af-9e9d4b179678}"), "address": "server_4.mysql.com:3060", "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_3) self.__group_2.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_2, self.__server_3) self.__options_5 = { "uuid": _uuid.UUID("{ee75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(2), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS prune_db") self.__server_5.exec_stmt("CREATE DATABASE prune_db") self.__server_5.exec_stmt("CREATE TABLE prune_db.prune_table" "(userID INT, name VARCHAR(30))") self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table " "VALUES(101, 'TEST 1')") self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table " "VALUES(202, 'TEST 2')") self.__options_6 = { "uuid": _uuid.UUID("{ff75a12a-98d1-414c-96af-9e9d4b179678}"), "address": "server_6.mysql.com:3060", "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__group_3 = Group("GROUPID3", "Third description.") Group.add(self.__group_3) self.__group_3.add_server(self.__server_5) self.__group_3.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_3, self.__server_5) group_4 = Group("GROUPID4", "4TH description.") Group.add(group_4) group_5 = Group("GROUPID5", "5TH description.") Group.add(group_5) group_6 = Group("GROUPID6", "6TH description.") Group.add(group_6) group_7 = Group("GROUPID7", "7TH description.") Group.add(group_7) group_8 = Group("GROUPID8", "8TH description.") Group.add(group_8) group_9 = Group("GROUPID9", "9TH description.") Group.add(group_9) group_10 = Group("GROUPID10", "10TH description.") Group.add(group_10) group_11 = Group("GROUPID11", "11TH description.") Group.add(group_11) group_12 = Group("GROUPID12", "12TH description.") Group.add(group_12) group_13 = Group("GROUPID13", "13TH description.") Group.add(group_13) group_14 = Group("GROUPID14", "14TH description.") Group.add(group_14) self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn() self.assertEquals(self.__shard_mapping_list, []) self.__shard_mapping_id_1 = ShardMapping.define("RANGE", "GROUPID10") self.__shard_mapping_id_2 = ShardMapping.define("RANGE", "GROUPID11") self.__shard_mapping_id_3 = ShardMapping.define("RANGE", "GROUPID12") #Test with sharding type values in lower case self.__shard_mapping_id_4 = ShardMapping.define("range", "GROUPID13") self.__shard_mapping_id_5 = ShardMapping.define("range", "GROUPID14") self.__shard_mapping_1 = \ ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID1") self.__shard_mapping_2 = \ ShardMapping.add(self.__shard_mapping_id_2, "db2.t2", "userID2") self.__shard_mapping_3 = \ ShardMapping.add(self.__shard_mapping_id_3, "db3.t3", "userID3") self.__shard_mapping_4 = \ ShardMapping.add(self.__shard_mapping_id_4, "db4.t4", "userID4") self.__shard_mapping_5 = \ ShardMapping.add(self.__shard_mapping_id_5, "prune_db.prune_table", "userID") self.__shard_id_1 = Shards.add("GROUPID1") self.__shard_id_2 = Shards.add("GROUPID10") self.__shard_id_3 = Shards.add("GROUPID11") self.__shard_id_4 = Shards.add("GROUPID4") self.__shard_id_5 = Shards.add("GROUPID5") self.__shard_id_6 = Shards.add("GROUPID6") self.__shard_id_7 = Shards.add("GROUPID7") self.__shard_id_8 = Shards.add("GROUPID8") self.__shard_id_9 = Shards.add("GROUPID9") self.__shard_id_10 = Shards.add("GROUPID2") self.__shard_id_11 = Shards.add("GROUPID3") self.__range_sharding_specification_1 = RangeShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, 0, self.__shard_id_1.shard_id) self.__range_sharding_specification_2 = RangeShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, 1001, self.__shard_id_2.shard_id) self.__range_sharding_specification_3 = RangeShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, 2001, self.__shard_id_3.shard_id) self.__range_sharding_specification_4 = RangeShardingSpecification.add( self.__shard_mapping_2.shard_mapping_id, 3001, self.__shard_id_4.shard_id) self.__range_sharding_specification_5 = RangeShardingSpecification.add( self.__shard_mapping_2.shard_mapping_id, 4001, self.__shard_id_5.shard_id) self.__range_sharding_specification_6 = RangeShardingSpecification.add( self.__shard_mapping_3.shard_mapping_id, 6001, self.__shard_id_6.shard_id) self.__range_sharding_specification_7 = RangeShardingSpecification.add( self.__shard_mapping_3.shard_mapping_id, 7001, self.__shard_id_7.shard_id) self.__range_sharding_specification_8 = RangeShardingSpecification.add( self.__shard_mapping_4.shard_mapping_id, 8001, self.__shard_id_8.shard_id) self.__range_sharding_specification_9 = RangeShardingSpecification.add( self.__shard_mapping_4.shard_mapping_id, 10001, self.__shard_id_9.shard_id) self.__range_sharding_specification_10 = RangeShardingSpecification.add( self.__shard_mapping_5.shard_mapping_id, 100, self.__shard_id_10.shard_id) self.__range_sharding_specification_11 = RangeShardingSpecification.add( self.__shard_mapping_5.shard_mapping_id, 201, self.__shard_id_11.shard_id) def tearDown(self): """Clean up the existing environment """ tests.utils.cleanup_environment() def test_fetch_shard_mapping(self): shard_mapping_1 = ShardMapping.fetch("db1.t1") shard_mapping_2 = ShardMapping.fetch("db2.t2") shard_mapping_3 = ShardMapping.fetch("db3.t3") shard_mapping_4 = ShardMapping.fetch("db4.t4") self.assertTrue( ShardingUtils.compare_shard_mapping(self.__shard_mapping_1, shard_mapping_1)) self.assertTrue( ShardingUtils.compare_shard_mapping(self.__shard_mapping_2, shard_mapping_2)) self.assertTrue( ShardingUtils.compare_shard_mapping(self.__shard_mapping_3, shard_mapping_3)) self.assertTrue( ShardingUtils.compare_shard_mapping(self.__shard_mapping_4, shard_mapping_4)) def test_fetch_sharding_scheme(self): range_sharding_specifications = RangeShardingSpecification.list(1) self.assertTrue( ShardingUtils.compare_range_specifications( range_sharding_specifications[0], self.__range_sharding_specification_1)) self.assertTrue( ShardingUtils.compare_range_specifications( range_sharding_specifications[1], self.__range_sharding_specification_2)) self.assertTrue( ShardingUtils.compare_range_specifications( range_sharding_specifications[2], self.__range_sharding_specification_3)) def test_lookup_sharding_scheme(self): r_spec_1 = \ RangeShardingSpecification.lookup(500, self.__shard_mapping_id_1, "RANGE") self.assertEqual(r_spec_1.shard_id, self.__shard_id_1.shard_id) r_spec_2 = \ RangeShardingSpecification.lookup(3500, self.__shard_mapping_id_2, "RANGE") self.assertEqual(r_spec_2.shard_id, self.__shard_id_4.shard_id) r_spec_3 = \ RangeShardingSpecification.lookup(6500, self.__shard_mapping_id_3, "RANGE") self.assertEqual(r_spec_3.shard_id, self.__shard_id_6.shard_id) def test_shard_mapping_list_mappings(self): shard_mappings = ShardMapping.list("RANGE") self.assertTrue( ShardingUtils.compare_shard_mapping(self.__shard_mapping_1, shard_mappings[0])) self.assertTrue( ShardingUtils.compare_shard_mapping(self.__shard_mapping_2, shard_mappings[1])) self.assertTrue( ShardingUtils.compare_shard_mapping(self.__shard_mapping_3, shard_mappings[2])) self.assertTrue( ShardingUtils.compare_shard_mapping(self.__shard_mapping_4, shard_mappings[3])) def test_shard_mapping_getters(self): self.assertEqual(self.__shard_mapping_1.table_name, "db1.t1") self.assertEqual(self.__shard_mapping_1.column_name, "userID1") self.assertEqual(self.__shard_mapping_1.type_name, "RANGE") self.assertEqual(self.__shard_mapping_1.shard_mapping_id, 1) self.assertEqual(self.__shard_mapping_1.global_group, "GROUPID10") def test_shard_mapping_remove(self): self.__range_sharding_specification_1.remove() self.__range_sharding_specification_2.remove() self.__range_sharding_specification_3.remove() self.__range_sharding_specification_4.remove() self.__range_sharding_specification_5.remove() self.__range_sharding_specification_6.remove() self.__range_sharding_specification_7.remove() self.__range_sharding_specification_8.remove() self.__range_sharding_specification_9.remove() self.__range_sharding_specification_10.remove() self.__shard_id_1.remove() self.__shard_id_2.remove() self.__shard_id_3.remove() self.__shard_id_4.remove() self.__shard_id_5.remove() self.__shard_id_6.remove() self.__shard_id_7.remove() self.__shard_id_8.remove() self.__shard_id_9.remove() self.__shard_id_10.remove() shard_mapping_1 = ShardMapping.fetch("db1.t1") shard_mapping_1.remove() def test_range_sharding_specification_getters(self): self.assertEqual( self.__range_sharding_specification_1.shard_mapping_id, 1) self.assertEqual(self.__range_sharding_specification_1.lower_bound, 0) self.assertEqual(self.__range_sharding_specification_1.shard_id, 1) def test_list_shard_mapping(self): expected_shard_mapping_list1 = [1, "RANGE", "GROUPID10"] expected_shard_mapping_list2 = [2, "RANGE", "GROUPID11"] expected_shard_mapping_list3 = [3, "RANGE", "GROUPID12"] expected_shard_mapping_list4 = [4, "RANGE", "GROUPID13"] expected_shard_mapping_list5 = [5, "RANGE", "GROUPID14"] obtained_shard_mapping_list = ShardMapping.list_shard_mapping_defn() self.assertEqual(set(expected_shard_mapping_list1), set(obtained_shard_mapping_list[0])) self.assertEqual(set(expected_shard_mapping_list2), set(obtained_shard_mapping_list[1])) self.assertEqual(set(expected_shard_mapping_list3), set(obtained_shard_mapping_list[2])) self.assertEqual(set(expected_shard_mapping_list4), set(obtained_shard_mapping_list[3])) self.assertEqual(set(expected_shard_mapping_list5), set(obtained_shard_mapping_list[4]))
class TestShardingPrune(tests.utils.TestCase): def setUp(self): """Creates the following topology for testing, GROUPID1 - localhost:13001, localhost:13002 - Global Group GROUPID2 - localhost:13003, localhost:13004 - shard 1 GROUPID3 - localhost:13005, localhost:13006 - shard 2 """ self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(0), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(1), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 71): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(101, 301): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(1001, 1201): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(10001, 10201): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(100001, 100201): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(2), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add(self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 71): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(101, 301): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(1001, 1201): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(10001, 10201): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(100001, 100201): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add(self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(3), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 71): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(101, 301): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(1001, 1201): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(10001, 10201): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(100001, 100201): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add(self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(4), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 71): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(101, 301): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(1001, 1201): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(10001, 10201): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(100001, 100201): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add(self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(5), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__server_6.connect() self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_6.exec_stmt("CREATE DATABASE db1") self.__server_6.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 71): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(101, 301): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(1001, 1201): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(10001, 10201): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(100001, 100201): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add(self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) status = self.proxy.sharding.create_definition("RANGE", "GROUPID1") self.check_xmlrpc_command_result(status, returns=1) status = self.proxy.sharding.add_table(1, "db1.t1", "userID") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_shard( 1, "GROUPID2/1,GROUPID3/101,GROUPID4/1001," "GROUPID5/10001,GROUPID6/100001", "ENABLED") self.check_xmlrpc_command_result(status) def test_prune_table_not_exists(self): """Delete the data in the database and verify that the prune does not fail once the database has been removed. """ self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1") status = self.proxy.sharding.prune_shard("db1.t1") def test_prune_shard(self): status = self.proxy.sharding.prune_shard("db1.t1") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.lookup_servers("db1.t1", 1, "LOCAL") info = self.check_xmlrpc_simple(status, {}, rowcount=1) shard_server = fetch_test_server(info['server_uuid']) shard_server.connect() rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 70) rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 70) rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 1) status = self.proxy.sharding.lookup_servers("db1.t1", 101, "LOCAL") info = self.check_xmlrpc_simple(status, {}, rowcount=1) shard_server = fetch_test_server(info['server_uuid']) shard_server.connect() rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 200) rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 300) rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 101) status = self.proxy.sharding.lookup_servers("db1.t1", 1202, "LOCAL") info = self.check_xmlrpc_simple(status, {}, rowcount=1) shard_server = fetch_test_server(info['server_uuid']) shard_server.connect() rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 200) rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 1200) rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 1001) status = self.proxy.sharding.lookup_servers("db1.t1", 11000, "LOCAL") info = self.check_xmlrpc_simple(status, {}, rowcount=1) shard_server = fetch_test_server(info['server_uuid']) shard_server.connect() rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 200) rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 10200) rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 10001) status = self.proxy.sharding.lookup_servers("db1.t1", 100100, "LOCAL") info = self.check_xmlrpc_simple(status, {}, rowcount=1) shard_server = fetch_test_server(info['server_uuid']) shard_server.connect() rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 200) rows = shard_server.exec_stmt("SELECT MAX(userID) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 100200) rows = shard_server.exec_stmt("SELECT MIN(userID) FROM db1.t1", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 100001) def tearDown(self): """Clean up the existing environment """ tests.utils.cleanup_environment()
class TestShardingPrune(tests.utils.TestCase): """Contains unit tests for testing the shard move operation and for verifying that the global server configuration remains constant after the shard move configuration. """ def check_xmlrpc_command_success(self, packet, expect, has_error): result = _xmlrpc._decode(packet) self.assertEqual(bool(result.error), has_error, "Had error '%s'" % result.error) # Check that it returned the expected result (if one is # expected), which is in the first result set (containing # execution result), first row (there is just one for # procedures), and last column (containing the result). if expect is not None: self.assertEqual(result.results[0][0][3], expect) # If the call was successful, check that there is at least 2 # result sets and that the second result set contain more than # zero jobs. if not has_error: self.assertTrue(len(result.results) > 1, str(result)) self.assertNotEqual(result.results[1].rowcount, 0, "had %d result sets" % len(result.results)) def setUp(self): """Creates the topology for testing. """ tests.utils.cleanup_environment() self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(0), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__server_1.connect() self.__server_1.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF") self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(1), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF") self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71, 10): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(101, 301, 10): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(1001, 1201, 10): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(10001, 10201, 10): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(100001, 100201, 10): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_2.exec_stmt("CREATE DATABASE db2") self.__server_2.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71, 10): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(101, 301, 10): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(1001, 1201, 10): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(10001, 10201, 10): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(100001, 100201, 10): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(2), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add(self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF") self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71, 10): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(101, 301, 10): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(1001, 1201, 10): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(10001, 10201, 10): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(100001, 100201, 10): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_3.exec_stmt("CREATE DATABASE db2") self.__server_3.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71, 10): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(101, 301, 10): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(1001, 1201, 10): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(10001, 10201, 10): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(100001, 100201, 10): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add(self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(3), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF") self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71, 10): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(101, 301, 10): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(1001, 1201, 10): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(10001, 10201, 10): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(100001, 100201, 10): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_4.exec_stmt("CREATE DATABASE db2") self.__server_4.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71, 10): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(101, 301, 10): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(1001, 1201, 10): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(10001, 10201, 10): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(100001, 100201, 10): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add(self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(4), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF") self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71, 10): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(101, 301, 10): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(1001, 1201, 10): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(10001, 10201, 10): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) for i in range(100001, 100201, 10): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_5.exec_stmt("CREATE DATABASE db2") self.__server_5.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71, 10): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(101, 301, 10): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(1001, 1201, 10): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(10001, 10201, 10): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) for i in range(100001, 100201, 10): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add(self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(5), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__server_6.connect() self.__server_6.exec_stmt("SET FOREIGN_KEY_CHECKS = OFF") self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add(self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) packet = self.proxy.sharding.create_definition("RANGE", "GROUPID1") self.check_xmlrpc_command_success(packet, 1, False) packet = self.proxy.sharding.add_table(1, "db1.t1", "userID") self.check_xmlrpc_command_success(packet, None, False) packet = self.proxy.sharding.add_table(1, "db2.t2", "userID") self.check_xmlrpc_command_success(packet, None, False) packet = self.proxy.sharding.add_shard( 1, "GROUPID2/1,GROUPID3/101,GROUPID4/1001,GROUPID5/10001", "ENABLED") self.check_xmlrpc_command_success(packet, None, False) packet = self.proxy.sharding.prune_shard("db1.t1") self.check_xmlrpc_command_success(packet, None, False) def test_move_shard_1(self): '''Test the move of shard 1 and the global server configuration after that. The test moves shard 1 from GROUPID2 to GROUPID6. After the move is done, it verifies the count on GROUPID6 to check that the group has all the tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. GROUPID2 should not have received the values since it has had the shard moved away from it. ''' row_cnt_shard_before_move_db1_t1 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_before_move_db2_t2 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_before_move_db1_t1 = \ int(row_cnt_shard_before_move_db1_t1[0][0]) row_cnt_shard_before_move_db2_t2 = \ int(row_cnt_shard_before_move_db2_t2[0][0]) self.assertEqual(row_cnt_shard_before_move_db1_t1, 7) self.assertEqual(row_cnt_shard_before_move_db2_t2, 7) packet = self.proxy.sharding.move_shard("1", "GROUPID6") self.check_xmlrpc_command_success(packet, None, False) row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_after_move_db1_t1 = \ int(row_cnt_shard_after_move_db1_t1[0][0]) row_cnt_shard_after_move_db2_t2 = \ int(row_cnt_shard_after_move_db2_t2[0][0]) self.assertEqual(row_cnt_shard_after_move_db1_t1, row_cnt_shard_before_move_db1_t1) self.assertEqual(row_cnt_shard_after_move_db2_t2, row_cnt_shard_before_move_db2_t2) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) try: #Verify that the data is not there in the first shard's old #group. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) raise Exception("Server should not be connected to global server") except _errors.DatabaseError: pass #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertEqual(global_table_count, 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the first shard's new #group. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_move_shard_2(self): '''Test the move of shard 2 and the global server configuration after that. The test moves shard 2 from GROUPID3 to GROUPID6. After the move is done, it verifies the count on GROUPID6 to check that the group has all the tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. GROUPID3 should not have received the values since it has had the shard moved away from it. ''' row_cnt_shard_before_move_db1_t1 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_before_move_db2_t2 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_before_move_db1_t1 = \ int(row_cnt_shard_before_move_db1_t1[0][0]) row_cnt_shard_before_move_db2_t2 = \ int(row_cnt_shard_before_move_db2_t2[0][0]) self.assertEqual(row_cnt_shard_before_move_db1_t1, 20) self.assertEqual(row_cnt_shard_before_move_db2_t2, 20) packet = self.proxy.sharding.move_shard("2", "GROUPID6") self.check_xmlrpc_command_success(packet, None, False) row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_after_move_db1_t1 = \ int(row_cnt_shard_after_move_db1_t1[0][0]) row_cnt_shard_after_move_db2_t2 = \ int(row_cnt_shard_after_move_db2_t2[0][0]) self.assertEqual(row_cnt_shard_after_move_db1_t1, row_cnt_shard_before_move_db1_t1) self.assertEqual(row_cnt_shard_after_move_db2_t2, row_cnt_shard_before_move_db2_t2) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) try: #Verify that the data is not there in the second shard's #old group. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) raise Exception("Server should not be connected to global server") except _errors.DatabaseError: pass #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertEqual(global_table_count, 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard's new #group. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_move_shard_3(self): '''Test the move of shard 3 and the global server configuration after that. The test moves shard 3 from GROUPID4 to GROUPID6. After the move is done, it verifies the count on GROUPID6 to check that the group has all the tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. GROUPID4 should not have received the values since it has had the shard moved away from it. ''' row_cnt_shard_before_move_db1_t1 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_before_move_db2_t2 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_before_move_db1_t1 = \ int(row_cnt_shard_before_move_db1_t1[0][0]) row_cnt_shard_before_move_db2_t2 = \ int(row_cnt_shard_before_move_db2_t2[0][0]) self.assertEqual(row_cnt_shard_before_move_db1_t1, 20) self.assertEqual(row_cnt_shard_before_move_db2_t2, 20) packet = self.proxy.sharding.move_shard("3", "GROUPID6") self.check_xmlrpc_command_success(packet, None, False) row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_after_move_db1_t1 = \ int(row_cnt_shard_after_move_db1_t1[0][0]) row_cnt_shard_after_move_db2_t2 = \ int(row_cnt_shard_after_move_db2_t2[0][0]) self.assertEqual(row_cnt_shard_after_move_db1_t1, row_cnt_shard_before_move_db1_t1) self.assertEqual(row_cnt_shard_after_move_db2_t2, row_cnt_shard_before_move_db2_t2) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) try: #Verify that the data is not there in the third shard's #original group. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) raise Exception("Server should not be connected to global server") except _errors.DatabaseError: pass #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertEqual(global_table_count, 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard #new group. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_move_shard_4(self): '''Test the move of shard 4 and the global server configuration after that. The test moves shard 4 from GROUPID5 to GROUPID6. After the move is done, it verifies the count on GROUPID6 to check that the group has all the tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. GROUPID5 should not have received the values since it has had the shard moved away from it. ''' row_cnt_shard_before_move_db1_t1 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_before_move_db2_t2 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_before_move_db1_t1 = \ int(row_cnt_shard_before_move_db1_t1[0][0]) row_cnt_shard_before_move_db2_t2 = \ int(row_cnt_shard_before_move_db2_t2[0][0]) self.assertEqual(row_cnt_shard_before_move_db1_t1, 40) self.assertEqual(row_cnt_shard_before_move_db2_t2, 40) packet = self.proxy.sharding.move_shard("4", "GROUPID6") self.check_xmlrpc_command_success(packet, None, False) row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch": True}) row_cnt_shard_after_move_db1_t1 = \ int(row_cnt_shard_after_move_db1_t1[0][0]) row_cnt_shard_after_move_db2_t2 = \ int(row_cnt_shard_after_move_db2_t2[0][0]) self.assertEqual(row_cnt_shard_after_move_db1_t1, row_cnt_shard_before_move_db1_t1) self.assertEqual(row_cnt_shard_after_move_db2_t2, row_cnt_shard_before_move_db2_t2) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) try: #Verify that the data is not there in the fourth shard's #original group. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) raise Exception("Server should not be connected to global server") except _errors.DatabaseError: pass #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertEqual(global_table_count, 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard's new #group. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch": True}) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def tearDown(self): """Clean up the existing environment """ tests.utils.cleanup_environment()
class TestShardingPrune(tests.utils.TestCase): """Contains unit tests for testing the shard split operation and for verifying that the global server configuration remains constant after the shard split configuration. """ def setUp(self): """Creates the topology for testing. """ tests.utils.cleanup_environment() self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__server_1.connect() self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID VARCHAR(20) PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("a"+str(i), i)) for i in range(101, 301): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("c"+str(i), i)) for i in range(10001, 10201): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("d"+str(i), i)) for i in range(100001, 100201): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("e"+str(i), i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_2.exec_stmt("CREATE DATABASE db2") self.__server_2.exec_stmt("CREATE TABLE db2.t2" "(userID VARCHAR(20), salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("a"+str(i), i)) for i in range(101, 301): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("c"+str(i), i)) for i in range(10001, 10201): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("d"+str(i), i)) for i in range(100001, 100201): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("e"+str(i), i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add( self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("a"+str(i), i)) for i in range(101, 301): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("c"+str(i), i)) for i in range(10001, 10201): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("d"+str(i), i)) for i in range(100001, 100201): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("e"+str(i), i)) self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_3.exec_stmt("CREATE DATABASE db2") self.__server_3.exec_stmt("CREATE TABLE db2.t2" "(userID VARCHAR(20), salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("a"+str(i), i)) for i in range(101, 301): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("c"+str(i), i)) for i in range(10001, 10201): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("d"+str(i), i)) for i in range(100001, 100201): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("e"+str(i), i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add( self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(3), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("a"+str(i), i)) for i in range(101, 301): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("c"+str(i), i)) for i in range(10001, 10201): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("d"+str(i), i)) for i in range(100001, 100201): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("e"+str(i), i)) self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_4.exec_stmt("CREATE DATABASE db2") self.__server_4.exec_stmt("CREATE TABLE db2.t2" "(userID VARCHAR(20), salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("a"+str(i), i)) for i in range(101, 301): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("c"+str(i), i)) for i in range(10001, 10201): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("d"+str(i), i)) for i in range(100001, 100201): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("e"+str(i), i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add( self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(4), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID VARCHAR(20)PRIMARY KEY, name VARCHAR(30))") for i in range(1, 71): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("a"+str(i), i)) for i in range(101, 301): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("c"+str(i), i)) for i in range(10001, 10201): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("d"+str(i), i)) for i in range(100001, 100201): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES('%s', 'TEST %s')" % ("e"+str(i), i)) self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_5.exec_stmt("CREATE DATABASE db2") self.__server_5.exec_stmt("CREATE TABLE db2.t2" "(userID VARCHAR(20), salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 71): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("a"+str(i), i)) for i in range(101, 301): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("b"+str(i), i)) for i in range(1001, 1201): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("c"+str(i), i)) for i in range(10001, 10201): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("d"+str(i), i)) for i in range(100001, 100201): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES('%s', %s)" % ("e"+str(i), i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add( self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(5), "user" : MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__server_6.connect() self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add( self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) status = self.proxy.sharding.create_definition("RANGE_STRING", "GROUPID1") self.check_xmlrpc_command_result(status, returns=1) status = self.proxy.sharding.add_table(1, "db1.t1", "userID") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_table(1, "db2.t2", "userID") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_shard( 1, "GROUPID2/a,GROUPID3/b,GROUPID4/c,GROUPID5/d", "ENABLED" ) self.check_xmlrpc_command_result(status) status = self.proxy.sharding.prune_shard("db1.t1") self.check_xmlrpc_command_result(status) def test_split_shard_4(self): '''Test the split of shard 4 and the global server configuration after that. The test splits shard 4 between GROUPID5 and GROUPID6. After the split is done, it verifies the count on GROUPID6 to check that the group has tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. ''' row_cnt_shard_before_split_db1_t1 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_before_split_db2_t2 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_before_split_db1_t1 = \ int(row_cnt_shard_before_split_db1_t1[0][0]) row_cnt_shard_before_split_db2_t2 = \ int(row_cnt_shard_before_split_db2_t2[0][0]) self.assertEqual(row_cnt_shard_before_split_db1_t1, 400) self.assertEqual(row_cnt_shard_before_split_db2_t2, 400) status = self.proxy.sharding.split_shard("4", "GROUPID6", "e") self.check_xmlrpc_command_result(status) self.__server_5.connect() row_cnt_shard_after_split_db1_t1_a = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_db1_t1_a = \ int(row_cnt_shard_after_split_db1_t1_a[0][0]) self.assertEqual(row_cnt_shard_after_split_db1_t1_a, 200) row_cnt_shard_after_split_db2_t2_a = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_db2_t2_a = \ int(row_cnt_shard_after_split_db2_t2_a[0][0]) self.assertEqual(row_cnt_shard_after_split_db2_t2_a, 200) row_cnt_shard_after_split_db1_t1_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_split_db1_t1_b = \ int(row_cnt_shard_after_split_db1_t1_b[0][0]) self.assertEqual(row_cnt_shard_after_split_db1_t1_b, 200) row_cnt_shard_after_split_db2_t2_b = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_split_db2_t2_b = \ int(row_cnt_shard_after_split_db2_t2_b[0][0]) self.assertEqual(row_cnt_shard_after_split_db2_t2_b, 200) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID VARCHAR(20), name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES('%s', 'TEST %s')" % (i, i)) time.sleep(5) #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def tearDown(self): """Clean up the existing environment """ tests.utils.cleanup_environment()
class TestSharding(unittest.TestCase): def setUp(self): """Configure the existing environment """ self.__options_1 = { "uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : "server_1.mysql.com:3060", } self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__options_2 = { "uuid" : _uuid.UUID("{aa75a12a-98d1-414c-96af-9e9d4b179678}"), "address" : "server_2.mysql.com:3060", } self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) self.__group_1.add_server(self.__server_2) self.__options_3 = { "uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add(self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS prune_db") self.__server_3.exec_stmt("CREATE DATABASE prune_db") self.__server_3.exec_stmt("CREATE TABLE prune_db.prune_table" "(userID INT, name VARCHAR(30))") self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table " "VALUES(101, 'TEST 1')") self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table " "VALUES(202, 'TEST 2')") self.__options_4 = { "uuid" : _uuid.UUID("{dd75a12a-98d1-414c-96af-9e9d4b179678}"), "address" : "server_4.mysql.com:3060", } self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_3) self.__group_2.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_2, self.__server_3) self.__options_5 = { "uuid" : _uuid.UUID("{ee75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS prune_db") self.__server_5.exec_stmt("CREATE DATABASE prune_db") self.__server_5.exec_stmt("CREATE TABLE prune_db.prune_table" "(userID INT, name VARCHAR(30))") self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table " "VALUES(101, 'TEST 1')") self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table " "VALUES(202, 'TEST 2')") self.__options_6 = { "uuid" : _uuid.UUID("{ff75a12a-98d1-414c-96af-9e9d4b179678}"), "address" : "server_6.mysql.com:3060", } self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__group_3 = Group("GROUPID3", "Third description.") Group.add(self.__group_3) self.__group_3.add_server(self.__server_5) self.__group_3.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_3, self.__server_5) group_4 = Group("GROUPID4", "4TH description.") Group.add(group_4) group_5 = Group("GROUPID5", "5TH description.") Group.add(group_5) group_6 = Group("GROUPID6", "6TH description.") Group.add(group_6) group_7 = Group("GROUPID7", "7TH description.") Group.add(group_7) group_8 = Group("GROUPID8", "8TH description.") Group.add(group_8) group_9 = Group("GROUPID9", "9TH description.") Group.add(group_9) group_10 = Group("GROUPID10", "10TH description.") Group.add(group_10) group_11 = Group("GROUPID11", "11TH description.") Group.add(group_11) group_12 = Group("GROUPID12", "12TH description.") Group.add(group_12) group_13 = Group("GROUPID13", "13TH description.") Group.add(group_13) group_14 = Group("GROUPID14", "14TH description.") Group.add(group_14) self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn() self.assertEquals( self.__shard_mapping_list, []) self.__shard_mapping_id_1 = ShardMapping.define("RANGE", "GROUPID10") self.__shard_mapping_id_2 = ShardMapping.define("RANGE", "GROUPID11") self.__shard_mapping_id_3 = ShardMapping.define("RANGE", "GROUPID12") #Test with sharding type values in lower case self.__shard_mapping_id_4 = ShardMapping.define("range", "GROUPID13") self.__shard_mapping_id_5 = ShardMapping.define("range", "GROUPID14") self.__shard_mapping_1 = \ ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID1") self.__shard_mapping_2 = \ ShardMapping.add(self.__shard_mapping_id_2, "db2.t2", "userID2") self.__shard_mapping_3 = \ ShardMapping.add(self.__shard_mapping_id_3, "db3.t3", "userID3") self.__shard_mapping_4 = \ ShardMapping.add(self.__shard_mapping_id_4, "db4.t4", "userID4") self.__shard_mapping_5 = \ ShardMapping.add(self.__shard_mapping_id_5, "prune_db.prune_table", "userID") self.__shard_id_1 = Shards.add("GROUPID1") self.__shard_id_2 = Shards.add("GROUPID10") self.__shard_id_3 = Shards.add("GROUPID11") self.__shard_id_4 = Shards.add("GROUPID4") self.__shard_id_5 = Shards.add("GROUPID5") self.__shard_id_6 = Shards.add("GROUPID6") self.__shard_id_7 = Shards.add("GROUPID7") self.__shard_id_8 = Shards.add("GROUPID8") self.__shard_id_9 = Shards.add("GROUPID9") self.__shard_id_10 = Shards.add("GROUPID2") self.__shard_id_11 = Shards.add("GROUPID3") self.__range_sharding_specification_1 = RangeShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, 0, self.__shard_id_1.shard_id ) self.__range_sharding_specification_2 = RangeShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, 1001, self.__shard_id_2.shard_id ) self.__range_sharding_specification_3 = RangeShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, 2001, self.__shard_id_3.shard_id ) self.__range_sharding_specification_4 = RangeShardingSpecification.add( self.__shard_mapping_2.shard_mapping_id, 3001, self.__shard_id_4.shard_id ) self.__range_sharding_specification_5 = RangeShardingSpecification.add( self.__shard_mapping_2.shard_mapping_id, 4001, self.__shard_id_5.shard_id ) self.__range_sharding_specification_6 = RangeShardingSpecification.add( self.__shard_mapping_3.shard_mapping_id, 6001, self.__shard_id_6.shard_id ) self.__range_sharding_specification_7 = RangeShardingSpecification.add( self.__shard_mapping_3.shard_mapping_id, 7001, self.__shard_id_7.shard_id ) self.__range_sharding_specification_8 = RangeShardingSpecification.add( self.__shard_mapping_4.shard_mapping_id, 8001, self.__shard_id_8.shard_id ) self.__range_sharding_specification_9 = RangeShardingSpecification.add( self.__shard_mapping_4.shard_mapping_id, 10001, self.__shard_id_9.shard_id ) self.__range_sharding_specification_10 = RangeShardingSpecification.add( self.__shard_mapping_5.shard_mapping_id, 100, self.__shard_id_10.shard_id) self.__range_sharding_specification_11 = RangeShardingSpecification.add( self.__shard_mapping_5.shard_mapping_id, 201, self.__shard_id_11.shard_id) def tearDown(self): """Clean up the existing environment """ self.__server_3.exec_stmt("DROP DATABASE IF EXISTS prune_db") self.__server_5.exec_stmt("DROP DATABASE IF EXISTS prune_db") tests.utils.cleanup_environment() def test_fetch_shard_mapping(self): shard_mapping_1 = ShardMapping.fetch("db1.t1") shard_mapping_2 = ShardMapping.fetch("db2.t2") shard_mapping_3 = ShardMapping.fetch("db3.t3") shard_mapping_4 = ShardMapping.fetch("db4.t4") self.assertTrue(ShardingUtils.compare_shard_mapping (self.__shard_mapping_1, shard_mapping_1)) self.assertTrue(ShardingUtils.compare_shard_mapping (self.__shard_mapping_2, shard_mapping_2)) self.assertTrue(ShardingUtils.compare_shard_mapping (self.__shard_mapping_3, shard_mapping_3)) self.assertTrue(ShardingUtils.compare_shard_mapping (self.__shard_mapping_4, shard_mapping_4)) def test_fetch_sharding_scheme(self): range_sharding_specifications = RangeShardingSpecification.list(1) self.assertTrue(ShardingUtils.compare_range_specifications( range_sharding_specifications[0], self.__range_sharding_specification_1)) self.assertTrue(ShardingUtils.compare_range_specifications( range_sharding_specifications[1], self.__range_sharding_specification_2)) self.assertTrue(ShardingUtils.compare_range_specifications( range_sharding_specifications[2], self.__range_sharding_specification_3)) def test_lookup_sharding_scheme(self): r_spec_1 = \ RangeShardingSpecification.lookup(500, self.__shard_mapping_id_1, "RANGE") self.assertEqual(r_spec_1.shard_id, self.__shard_id_1.shard_id) r_spec_2 = \ RangeShardingSpecification.lookup(3500, self.__shard_mapping_id_2, "RANGE") self.assertEqual(r_spec_2.shard_id, self.__shard_id_4.shard_id) r_spec_3 = \ RangeShardingSpecification.lookup(6500, self.__shard_mapping_id_3, "RANGE") self.assertEqual(r_spec_3.shard_id, self.__shard_id_6.shard_id) def test_shard_mapping_list_mappings(self): shard_mappings = ShardMapping.list("RANGE") self.assertTrue(ShardingUtils.compare_shard_mapping (self.__shard_mapping_1, shard_mappings[0])) self.assertTrue(ShardingUtils.compare_shard_mapping (self.__shard_mapping_2, shard_mappings[1])) self.assertTrue(ShardingUtils.compare_shard_mapping (self.__shard_mapping_3, shard_mappings[2])) self.assertTrue(ShardingUtils.compare_shard_mapping (self.__shard_mapping_4, shard_mappings[3])) def test_shard_mapping_getters(self): self.assertEqual(self.__shard_mapping_1.table_name, "db1.t1") self.assertEqual(self.__shard_mapping_1.column_name, "userID1") self.assertEqual(self.__shard_mapping_1.type_name, "RANGE") self.assertEqual(self.__shard_mapping_1.shard_mapping_id, 1) self.assertEqual(self.__shard_mapping_1.global_group, "GROUPID10") def test_shard_mapping_remove(self): self.__range_sharding_specification_1.remove() self.__range_sharding_specification_2.remove() self.__range_sharding_specification_3.remove() self.__range_sharding_specification_4.remove() self.__range_sharding_specification_5.remove() self.__range_sharding_specification_6.remove() self.__range_sharding_specification_7.remove() self.__range_sharding_specification_8.remove() self.__range_sharding_specification_9.remove() self.__range_sharding_specification_10.remove() self.__shard_id_1.remove() self.__shard_id_2.remove() self.__shard_id_3.remove() self.__shard_id_4.remove() self.__shard_id_5.remove() self.__shard_id_6.remove() self.__shard_id_7.remove() self.__shard_id_8.remove() self.__shard_id_9.remove() self.__shard_id_10.remove() shard_mapping_1 = ShardMapping.fetch("db1.t1") shard_mapping_1.remove() def test_range_sharding_specification_getters(self): self.assertEqual( self.__range_sharding_specification_1.shard_mapping_id, 1) self.assertEqual(self.__range_sharding_specification_1.lower_bound, 0) self.assertEqual(self.__range_sharding_specification_1.shard_id, 1) def test_list_shard_mapping(self): expected_shard_mapping_list1 = [1, "RANGE", "GROUPID10"] expected_shard_mapping_list2 = [2, "RANGE", "GROUPID11"] expected_shard_mapping_list3 = [3, "RANGE", "GROUPID12"] expected_shard_mapping_list4 = [4, "RANGE", "GROUPID13"] expected_shard_mapping_list5 = [5, "RANGE", "GROUPID14"] obtained_shard_mapping_list = ShardMapping.list_shard_mapping_defn() self.assertEqual( set(expected_shard_mapping_list1), set(obtained_shard_mapping_list[0]) ) self.assertEqual( set(expected_shard_mapping_list2), set(obtained_shard_mapping_list[1]) ) self.assertEqual( set(expected_shard_mapping_list3), set(obtained_shard_mapping_list[2]) ) self.assertEqual( set(expected_shard_mapping_list4), set(obtained_shard_mapping_list[3]) ) self.assertEqual( set(expected_shard_mapping_list5), set(obtained_shard_mapping_list[4]) )
class TestShardingServices(unittest.TestCase): def assertStatus(self, status, expect): items = (item['diagnosis'] for item in status[1] if item['diagnosis']) self.assertEqual(status[1][-1]["success"], expect, "\n".join(items)) def setUp(self): """Configure the existing environment """ self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add( self.__server_3) self.__group_3 = Group("GROUPID3", "Third description.") Group.add( self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(3), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add( self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_4.exec_stmt("CREATE DATABASE db2") self.__server_4.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__options_5 = { "uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(4), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add( self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_5.exec_stmt("CREATE DATABASE db2") self.__server_5.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__options_6 = { "uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(5), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add( self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) status = self.proxy.sharding.create_definition("HASH", "GROUPID1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_define_shard_mapping).") self.assertEqual(status[2], 1) status = self.proxy.sharding.create_definition("RANGE", "GROUPID1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_define_shard_mapping).") self.assertEqual(status[2], 2) status = self.proxy.sharding.create_definition("HASH", "GROUPID1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_define_shard_mapping).") self.assertEqual(status[2], 3) status = self.proxy.sharding.add_table(1, "db1.t1", "userID1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_table(2, "db2.t2", "userID2") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_table(3, "db3.t3", "userID3") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_shard(1, "GROUPID2,GROUPID3", "ENABLED") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard).") status = self.proxy.sharding.add_shard(2, "GROUPID4/0,GROUPID5/101", "ENABLED") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard).") status = self.proxy.sharding.add_shard(3, "GROUPID6", "ENABLED") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard).") def test_list_shard_defns(self): expected_shard_mapping_list1 = [ [1, "HASH", "GROUPID1"], [2, "RANGE", "GROUPID1"], [3, "HASH", "GROUPID1"] ] status = self.proxy.sharding.list_definitions() self.assertEqual(status[0], True) self.assertEqual(status[1], "") obtained_shard_mapping_list = status[2] self.assertEqual(set(expected_shard_mapping_list1[0]), set(obtained_shard_mapping_list[0])) self.assertEqual(set(expected_shard_mapping_list1[1]), set(obtained_shard_mapping_list[1])) self.assertEqual(set(expected_shard_mapping_list1[2]), set(obtained_shard_mapping_list[2])) def test_list_shard_mappings(self): status = self.proxy.sharding.list_tables("HASH") self.assertEqual(status[0], True) self.assertEqual(status[1], "") self.assertEqual(status[2], [ {"shard_mapping_id":1, "table_name":"db1.t1", "column_name":"userID1", "type_name":"HASH", "global_group":"GROUPID1"}, {"shard_mapping_id":3, "table_name":"db3.t3", "column_name":"userID3", "type_name":"HASH", "global_group":"GROUPID1"} ]) status = self.proxy.sharding.list_tables("RANGE") self.assertEqual(status[0], True) self.assertEqual(status[1], "") self.assertEqual(status[2], [ {"shard_mapping_id":2, "table_name":"db2.t2", "column_name":"userID2", "type_name":"RANGE", "global_group":"GROUPID1"} ]) def test_shard_prune(self): status = self.proxy.sharding.prune_shard("db2.t2") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables).") status = self.proxy.sharding.lookup_servers("db2.t2", 1, "LOCAL") self.assertEqual(status[0], True) self.assertEqual(status[1], "") obtained_server_list = status[2] shard_uuid = obtained_server_list[0][0] shard_server = MySQLServer.fetch(shard_uuid) shard_server.connect() rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 100) rows = shard_server.exec_stmt( "SELECT MAX(userID2) FROM db2.t2", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 100) rows = shard_server.exec_stmt( "SELECT MIN(userID2) FROM db2.t2", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 1) status = self.proxy.sharding.lookup_servers("db2.t2", 101, "LOCAL") self.assertEqual(status[0], True) self.assertEqual(status[1], "") obtained_server_list = status[2] shard_uuid = obtained_server_list[0][0] shard_server = MySQLServer.fetch(shard_uuid) shard_server.connect() rows = shard_server.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 100) rows = shard_server.exec_stmt( "SELECT MAX(userID2) FROM db2.t2", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 200) rows = shard_server.exec_stmt( "SELECT MIN(userID2) FROM db2.t2", {"fetch" : True}) self.assertTrue(int(rows[0][0]) == 101) def tearDown(self): """Clean up the existing environment """ self.proxy.sharding.disable_shard(1) self.proxy.sharding.remove_shard(1) self.proxy.sharding.disable_shard(2) self.proxy.sharding.remove_shard(2) self.proxy.sharding.disable_shard(3) self.proxy.sharding.remove_shard(3) self.proxy.sharding.disable_shard(4) self.proxy.sharding.remove_shard(4) self.proxy.sharding.disable_shard(5) self.proxy.sharding.remove_shard(5) tests.utils.cleanup_environment() tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestShardingServices(tests.utils.TestCase): def setUp(self): """Configure the existing environment """ self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(0), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(1), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(2), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add(self.__server_3) self.__group_3 = Group("GROUPID3", "Third description.") Group.add(self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(3), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add(self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_4.exec_stmt("CREATE DATABASE db2") self.__server_4.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__options_5 = { "uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(4), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add(self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_5.exec_stmt("CREATE DATABASE db2") self.__server_5.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__options_6 = { "uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(5), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add(self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) status = self.proxy.sharding.create_definition("HASH", "GROUPID1") self.check_xmlrpc_command_result(status, returns=1) status = self.proxy.sharding.create_definition("RANGE", "GROUPID1") self.check_xmlrpc_command_result(status, returns=2) status = self.proxy.sharding.create_definition("HASH", "GROUPID1") self.check_xmlrpc_command_result(status, returns=3) status = self.proxy.sharding.add_table(1, "db1.t1", "userID1") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_table(2, "db2.t2", "userID2") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_table(3, "db3.t3", "userID3") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_shard(1, "GROUPID2,GROUPID3", "ENABLED") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_shard(2, "GROUPID4/0,GROUPID5/101", "ENABLED") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.add_shard(3, "GROUPID6", "ENABLED") self.check_xmlrpc_command_result(status) def test_list_shard_defns(self): expected_shard_mapping_list1 = tests.utils.make_mapping_result( [[1, "HASH", "GROUPID1"], [2, "RANGE", "GROUPID1"], [3, "HASH", "GROUPID1"]]) status = self.proxy.sharding.list_definitions() self.check_xmlrpc_result(status, expected_shard_mapping_list1) def test_list_shard_mappings(self): expected_hash = tests.utils.make_shard_mapping_list_result([ [1, 'HASH', "db1.t1", "GROUPID1", "userID1"], [3, 'HASH', "db3.t3", "GROUPID1", "userID3"], ]) status = self.proxy.sharding.list_tables("HASH") self.check_xmlrpc_result(status, expected_hash) expected_range = tests.utils.make_shard_mapping_list_result([ [2, "RANGE", "db2.t2", "GROUPID1", "userID2"], ]) status = self.proxy.sharding.list_tables("RANGE") self.check_xmlrpc_result(status, expected_range) def test_shard_prune(self): status = self.proxy.sharding.prune_shard("db2.t2") self.check_xmlrpc_command_result(status) status = self.proxy.sharding.lookup_servers("db2.t2", 1, "LOCAL") info = self.check_xmlrpc_simple(status, {}) shard_uuid = info['server_uuid'] shard_server = fetch_test_server(shard_uuid) shard_server.connect() rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db2.t2", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 100) rows = shard_server.exec_stmt("SELECT MAX(userID2) FROM db2.t2", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 100) rows = shard_server.exec_stmt("SELECT MIN(userID2) FROM db2.t2", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 1) status = self.proxy.sharding.lookup_servers("db2.t2", 101, "LOCAL") info = self.check_xmlrpc_simple(status, {}) shard_uuid = info['server_uuid'] shard_server = fetch_test_server(shard_uuid) shard_server.connect() rows = shard_server.exec_stmt("SELECT COUNT(*) FROM db2.t2", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 100) rows = shard_server.exec_stmt("SELECT MAX(userID2) FROM db2.t2", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 200) rows = shard_server.exec_stmt("SELECT MIN(userID2) FROM db2.t2", {"fetch": True}) self.assertTrue(int(rows[0][0]) == 101) def tearDown(self): """Clean up the existing environment """ tests.utils.cleanup_environment()
class TestHashMoveGlobal(unittest.TestCase): """Contains unit tests for testing the shard move operation and for verifying that the global server configuration remains constant after the shard move configuration. """ def assertStatus(self, status, expect): items = (item['diagnosis'] for item in status[1] if item['diagnosis']) self.assertEqual(status[1][-1]["success"], expect, "\n".join(items)) def setUp(self): """Configure the existing environment """ tests.utils.cleanup_environment() self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__server_1.connect() self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_2.exec_stmt("CREATE DATABASE db2") self.__server_2.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add( self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_3.exec_stmt("CREATE DATABASE db2") self.__server_3.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add( self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(3), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_4.exec_stmt("CREATE DATABASE db2") self.__server_4.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add( self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(4), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID INT PRIMARY KEY, name VARCHAR(30))") for i in range(1, 1001): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_5.exec_stmt("CREATE DATABASE db2") self.__server_5.exec_stmt("CREATE TABLE db2.t2" "(userID INT, salary INT, " "CONSTRAINT FOREIGN KEY(userID) " "REFERENCES db1.t1(userID))") for i in range(1, 1001): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, %s)" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add( self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(5), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add( self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) status = self.proxy.sharding.create_definition("HASH", "GROUPID1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_define_shard_mapping).") self.assertEqual(status[2], 1) status = self.proxy.sharding.add_table(1, "db1.t1", "userID") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_table(1, "db2.t2", "userID") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_shard( 1, "GROUPID2,GROUPID3,GROUPID4,GROUPID5", "ENABLED" ) self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard).") status = self.proxy.sharding.prune_shard("db1.t1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables).") def test_move_shard_1(self): '''Test the move of shard 1 and the global server configuration after that. The test moves shard 1 from GROUPID2 to GROUPID6. After the move is done, it verifies the count on GROUPID6 to check that the group has all the tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. GROUPID2 should not have received the values since it has had the shard moved away from it. ''' row_cnt_shard_before_move_db1_t1 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_before_move_db2_t2 = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_before_move_db1_t1 = \ int(row_cnt_shard_before_move_db1_t1[0][0]) row_cnt_shard_before_move_db2_t2 = \ int(row_cnt_shard_before_move_db2_t2[0][0]) status = self.proxy.sharding.move_shard("1", "GROUPID6") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_setup_resharding_switch).") self.__server_6.connect() row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_move_db1_t1 = \ int(row_cnt_shard_after_move_db1_t1[0][0]) row_cnt_shard_after_move_db2_t2 = \ int(row_cnt_shard_after_move_db2_t2[0][0]) self.assertTrue( row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1 ) self.assertTrue( row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2 ) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) try: #Verify that the data is not there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) raise Exception("Server should not be connected to global server") except _errors.DatabaseError: pass #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_move_shard_2(self): '''Test the move of shard 2 and the global server configuration after that. The test moves shard 2 from GROUPID3 to GROUPID6. After the move is done, it verifies the count on GROUPID6 to check that the group has all the tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. GROUPID3 should not have received the values since it has had the shard moved away from it. ''' row_cnt_shard_before_move_db1_t1 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_before_move_db2_t2 = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_before_move_db1_t1 = \ int(row_cnt_shard_before_move_db1_t1[0][0]) row_cnt_shard_before_move_db2_t2 = \ int(row_cnt_shard_before_move_db2_t2[0][0]) status = self.proxy.sharding.move_shard("2", "GROUPID6") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_setup_resharding_switch).") self.__server_6.connect() row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_move_db1_t1 = \ int(row_cnt_shard_after_move_db1_t1[0][0]) row_cnt_shard_after_move_db2_t2 = \ int(row_cnt_shard_after_move_db2_t2[0][0]) self.assertTrue( row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1 ) self.assertTrue( row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2 ) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) try: #Verify that the data is not there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) raise Exception("Server should not be connected to global server") except _errors.DatabaseError: pass #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fifth shard. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_move_shard_3(self): '''Test the move of shard 3 and the global server configuration after that. The test moves shard 3 from GROUPID4 to GROUPID6. After the move is done, it verifies the count on GROUPID6 to check that the group has all the tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. GROUPID4 should not have received the values since it has had the shard moved away from it. ''' row_cnt_shard_before_move_db1_t1 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_before_move_db2_t2 = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_before_move_db1_t1 = \ int(row_cnt_shard_before_move_db1_t1[0][0]) row_cnt_shard_before_move_db2_t2 = \ int(row_cnt_shard_before_move_db2_t2[0][0]) status = self.proxy.sharding.move_shard("3", "GROUPID6") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_setup_resharding_switch).") self.__server_6.connect() row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_move_db1_t1 = \ int(row_cnt_shard_after_move_db1_t1[0][0]) row_cnt_shard_after_move_db2_t2 = \ int(row_cnt_shard_after_move_db2_t2[0][0]) self.assertTrue( row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1 ) self.assertTrue( row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2 ) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) try: #Verify that the data is not there in the third shard's #original group. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) raise Exception("Server should not be connected to global server") except _errors.DatabaseError: pass #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard #new group. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def test_move_shard_4(self): '''Test the move of shard 4 and the global server configuration after that. The test moves shard 4 from GROUPID5 to GROUPID6. After the move is done, it verifies the count on GROUPID6 to check that the group has all the tuples from the earlier group. Now it fires an INSERT on the global group and verifies that all the shards have got the inserted tuples. GROUPID5 should not have received the values since it has had the shard moved away from it. ''' row_cnt_shard_before_move_db1_t1 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_before_move_db2_t2 = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_before_move_db1_t1 = \ int(row_cnt_shard_before_move_db1_t1[0][0]) row_cnt_shard_before_move_db2_t2 = \ int(row_cnt_shard_before_move_db2_t2[0][0]) status = self.proxy.sharding.move_shard("4", "GROUPID6") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_setup_resharding_switch).") self.__server_6.connect() row_cnt_shard_after_move_db1_t1 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_after_move_db2_t2 = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM db2.t2", {"fetch" : True} ) row_cnt_shard_after_move_db1_t1 = \ int(row_cnt_shard_after_move_db1_t1[0][0]) row_cnt_shard_after_move_db2_t2 = \ int(row_cnt_shard_after_move_db2_t2[0][0]) self.assertTrue( row_cnt_shard_before_move_db1_t1 == row_cnt_shard_after_move_db1_t1 ) self.assertTrue( row_cnt_shard_before_move_db2_t2 == row_cnt_shard_after_move_db2_t2 ) #Enter data into the global server self.__server_1.exec_stmt("DROP DATABASE IF EXISTS global") self.__server_1.exec_stmt("CREATE DATABASE global") self.__server_1.exec_stmt("CREATE TABLE global.global_table" "(userID INT, name VARCHAR(30))") for i in range(1, 11): self.__server_1.exec_stmt("INSERT INTO global.global_table " "VALUES(%s, 'TEST %s')" % (i, i)) time.sleep(5) try: #Verify that the data is not there in the fourth shard's #original group. global_table_count = self.__server_5.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) raise Exception("Server should not be connected to global server") except _errors.DatabaseError: pass #Verify that the data is there in the first shard. global_table_count = self.__server_2.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the second shard. global_table_count = self.__server_3.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the third shard. global_table_count = self.__server_4.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) #Verify that the data is there in the fourth shard's new #group. global_table_count = self.__server_6.exec_stmt( "SELECT COUNT(*) FROM global.global_table", {"fetch" : True} ) global_table_count = int(global_table_count[0][0]) self.assertTrue(global_table_count == 10) def tearDown(self): """Clean up the existing environment """ self.proxy.sharding.disable_shard(1) self.proxy.sharding.remove_shard(1) self.proxy.sharding.disable_shard(2) self.proxy.sharding.remove_shard(2) self.proxy.sharding.disable_shard(3) self.proxy.sharding.remove_shard(3) self.proxy.sharding.disable_shard(4) self.proxy.sharding.remove_shard(4) self.proxy.sharding.disable_shard(5) self.proxy.sharding.remove_shard(5) tests.utils.cleanup_environment() tests.utils.teardown_xmlrpc(self.manager, self.proxy)
class TestShardingServices(unittest.TestCase): def assertStatus(self, status, expect): items = (item['diagnosis'] for item in status[1] if item['diagnosis']) self.assertEqual(status[1][-1]["success"], expect, "\n".join(items)) def setUp(self): """Configure the existing environment """ tests.utils.cleanup_environment() self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID1 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_2.exec_stmt("CREATE DATABASE db2") self.__server_2.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(201, 401): self.__server_2.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_2.exec_stmt("CREATE DATABASE db3") self.__server_2.exec_stmt("CREATE TABLE db3.t3" "(userID3 INT, name VARCHAR(30))") for i in range(401, 601): self.__server_2.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add( self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID1 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_3.exec_stmt("CREATE DATABASE db2") self.__server_3.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(201, 401): self.__server_3.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_3.exec_stmt("CREATE DATABASE db3") self.__server_3.exec_stmt("CREATE TABLE db3.t3" "(userID3 INT, name VARCHAR(30))") for i in range(401, 601): self.__server_3.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add( self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(3), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID1 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_4.exec_stmt("CREATE DATABASE db2") self.__server_4.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(201, 401): self.__server_4.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_4.exec_stmt("CREATE DATABASE db3") self.__server_4.exec_stmt("CREATE TABLE db3.t3" "(userID3 INT, name VARCHAR(30))") for i in range(401, 601): self.__server_4.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add( self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(4), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID1 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_5.exec_stmt("CREATE DATABASE db2") self.__server_5.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(201, 401): self.__server_5.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_5.exec_stmt("CREATE DATABASE db3") self.__server_5.exec_stmt("CREATE TABLE db3.t3" "(userID3 INT, name VARCHAR(30))") for i in range(401, 601): self.__server_5.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add( self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(5), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__server_6.connect() self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_6.exec_stmt("CREATE DATABASE db1") self.__server_6.exec_stmt("CREATE TABLE db1.t1" "(userID1 INT, name VARCHAR(30))") for i in range(1, 201): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db2") self.__server_6.exec_stmt("CREATE DATABASE db2") self.__server_6.exec_stmt("CREATE TABLE db2.t2" "(userID2 INT, name VARCHAR(30))") for i in range(201, 401): self.__server_6.exec_stmt("INSERT INTO db2.t2 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db3") self.__server_6.exec_stmt("CREATE DATABASE db3") self.__server_6.exec_stmt("CREATE TABLE db3.t3" "(userID3 INT, name VARCHAR(30))") for i in range(401, 601): self.__server_6.exec_stmt("INSERT INTO db3.t3 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add( self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) status = self.proxy.sharding.create_definition("HASH", "GROUPID1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_define_shard_mapping).") self.assertEqual(status[2], 1) status = self.proxy.sharding.add_table(1, "db1.t1", "userID1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_table(1, "db2.t2", "userID2") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_table(1, "db3.t3", "userID3") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard_mapping).") status = self.proxy.sharding.add_shard( 1, "GROUPID2,GROUPID3,GROUPID4,GROUPID5,GROUPID6", "ENABLED" ) self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_add_shard).") status = self.proxy.sharding.prune_shard("db1.t1") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables).") status = self.proxy.sharding.prune_shard("db2.t2") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables).") status = self.proxy.sharding.prune_shard("db3.t3") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables).") def tearDown(self): """Clean up the existing environment """ self.proxy.sharding.disable_shard(1) self.proxy.sharding.remove_shard(1) self.proxy.sharding.disable_shard(2) self.proxy.sharding.remove_shard(2) self.proxy.sharding.disable_shard(3) self.proxy.sharding.remove_shard(3) self.proxy.sharding.disable_shard(4) self.proxy.sharding.remove_shard(4) self.proxy.sharding.disable_shard(5) self.proxy.sharding.remove_shard(5) tests.utils.cleanup_environment() tests.utils.teardown_xmlrpc(self.manager, self.proxy) def test_prune_lookup_shard1(self): '''Verify that after the prune the lookup of any pruned value in the shard results in looking up the same shard. ''' self.proxy.sharding.prune_shard("db1.t1") rows = self.__server_2.exec_stmt("SELECT userID1 FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_1.shard_id, 1 ) rows = self.__server_3.exec_stmt( "SELECT userID1 FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_2.shard_id, 2 ) rows = self.__server_4.exec_stmt( "SELECT userID1 FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_3.shard_id, 3 ) rows = self.__server_5.exec_stmt( "SELECT userID1 FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_4 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_4.shard_id, 4 ) rows = self.__server_6.exec_stmt( "SELECT userID1 FROM db1.t1", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_5.shard_id, 5 ) def test_prune_lookup_shard2(self): '''Verify that after the prune the lookup of any pruned value in the shard results in looking up the same shard. ''' self.proxy.sharding.prune_shard("db2.t2") rows = self.__server_2.exec_stmt("SELECT userID2 FROM db2.t2", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_1.shard_id, 1 ) rows = self.__server_3.exec_stmt( "SELECT userID2 FROM db2.t2", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_2.shard_id, 2 ) rows = self.__server_4.exec_stmt( "SELECT userID2 FROM db2.t2", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_3.shard_id, 3 ) rows = self.__server_5.exec_stmt( "SELECT userID2 FROM db2.t2", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_4 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_4.shard_id, 4 ) rows = self.__server_6.exec_stmt( "SELECT userID2 FROM db2.t2", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_5.shard_id, 5 ) def test_prune_lookup_shard3(self): '''Verify that after the prune the lookup of any pruned value in the shard results in looking up the same shard. ''' self.proxy.sharding.prune_shard("db3.t3") rows = self.__server_2.exec_stmt("SELECT userID3 FROM db3.t3", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_1 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_1.shard_id, 1 ) rows = self.__server_3.exec_stmt( "SELECT userID3 FROM db3.t3", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_2 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_2.shard_id, 2 ) rows = self.__server_4.exec_stmt( "SELECT userID3 FROM db3.t3", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_3 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_3.shard_id, 3 ) rows = self.__server_5.exec_stmt( "SELECT userID3 FROM db3.t3", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_4 = HashShardingSpecification.lookup( val, 1, "HASH" ) self.assertEqual( hash_sharding_spec_4.shard_id, 4 ) rows = self.__server_6.exec_stmt( "SELECT userID3 FROM db3.t3", {"fetch" : True}) for val in rows[0:len(rows)][0]: hash_sharding_spec_5 = HashShardingSpecification.lookup(val, 1, "HASH") self.assertEqual( hash_sharding_spec_5.shard_id, 5 ) def test_list_shard_mappings(self): expected_shard_mapping_list1 = [1, "HASH", "GROUPID1"] status = self.proxy.sharding.list_definitions() self.assertEqual(status[0], True) self.assertEqual(status[1], "") obtained_shard_mapping_list = status[2] self.assertEqual(set(expected_shard_mapping_list1), set(obtained_shard_mapping_list[0])) def test_lookup_shard_mapping(self): status = self.proxy.sharding.lookup_table("db1.t1") self.assertEqual(status[0], True) self.assertEqual(status[1], "") self.assertEqual(status[2], {"shard_mapping_id":1, "table_name":"db1.t1", "column_name":"userID1", "type_name":"HASH", "global_group":"GROUPID1"}) status = self.proxy.sharding.lookup_table("db2.t2") self.assertEqual(status[0], True) self.assertEqual(status[1], "") self.assertEqual(status[2], {"shard_mapping_id":1, "table_name":"db2.t2", "column_name":"userID2", "type_name":"HASH", "global_group":"GROUPID1"})