def test_hash_remove(self): """Test the removal of hash shards. """ hash_sharding_specification_1 = HashShardingSpecification.fetch(1) hash_sharding_specification_2 = HashShardingSpecification.fetch(2) hash_sharding_specification_3 = HashShardingSpecification.fetch(3) hash_sharding_specification_4 = HashShardingSpecification.fetch(4) hash_sharding_specification_5 = HashShardingSpecification.fetch(5) hash_sharding_specification_1.remove() hash_sharding_specification_2.remove() hash_sharding_specification_3.remove() hash_sharding_specification_4.remove() hash_sharding_specification_5.remove() self.__shard_1.remove() self.__shard_2.remove() self.__shard_3.remove() self.__shard_4.remove() self.__shard_5.remove() for i in range(0, 10): hash_sharding_spec = HashShardingSpecification.lookup( i, self.__shard_mapping_id_1, "HASH" ) self.assertEqual(hash_sharding_spec, None)
def test_fetch_sharding_scheme(self): """Test the fetch method of the HASH sharding scheme. """ hash_sharding_specification_1 = HashShardingSpecification.fetch(1) hash_sharding_specification_2 = HashShardingSpecification.fetch(2) hash_sharding_specification_3 = HashShardingSpecification.fetch(3) hash_sharding_specification_4 = HashShardingSpecification.fetch(4) hash_sharding_specification_5 = HashShardingSpecification.fetch(5) hash_sharding_specifications = HashShardingSpecification.list(1) #list does not return the hashing specifications in order of shard_id, #hence a direct comparison is not posssible. self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_1, hash_sharding_specifications)) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_2, hash_sharding_specifications)) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_3, hash_sharding_specifications)) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_4, hash_sharding_specifications)) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_5, hash_sharding_specifications))
def test_fetch_sharding_scheme(self): """Test the fetch method of the HASH sharding scheme. """ hash_sharding_specification_1 = HashShardingSpecification.fetch(1) hash_sharding_specification_2 = HashShardingSpecification.fetch(2) hash_sharding_specification_3 = HashShardingSpecification.fetch(3) hash_sharding_specification_4 = HashShardingSpecification.fetch(4) hash_sharding_specification_5 = HashShardingSpecification.fetch(5) hash_sharding_specifications = HashShardingSpecification.list(1) #list does not return the hashing specifications in order of shard_id, #hence a direct comparison is not posssible. self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_1, hash_sharding_specifications ) ) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_2, hash_sharding_specifications ) ) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_3, hash_sharding_specifications ) ) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_4, hash_sharding_specifications ) ) self.assertTrue( self.hash_sharding_specification_in_list( hash_sharding_specification_5, hash_sharding_specifications ) )
def verify_and_fetch_shard(shard_id): """Find out if the shard_id exists and return the sharding specification for it. If it does not exist throw an exception. :param shard_id: The ID for the shard whose specification needs to be fetched. :return: The sharding specification class representing the shard ID. :raises: ShardingError if the shard ID is not found. """ #Here the underlying sharding specification might be a RANGE #or a HASH. The type of sharding specification is obtained from the #shard mapping. range_sharding_spec = RangeShardingSpecification.fetch(shard_id) if range_sharding_spec is None: raise _errors.ShardingError(SHARD_NOT_FOUND % (shard_id, )) #Fetch the shard mappings and use them to find the type of sharding #scheme. shard_mappings = ShardMapping.fetch_by_id( range_sharding_spec.shard_mapping_id ) if shard_mappings is None: raise _errors.ShardingError( SHARD_MAPPING_NOT_FOUND % ( range_sharding_spec.shard_mapping_id, ) ) #Fetch the shard mapping definition. There is only one shard mapping #definition associated with all of the shard mappings. shard_mapping_defn = ShardMapping.fetch_shard_mapping_defn( range_sharding_spec.shard_mapping_id ) if shard_mapping_defn is None: raise _errors.ShardingError( SHARD_MAPPING_DEFN_NOT_FOUND % ( range_sharding_spec.shard_mapping_id, ) ) shard = Shards.fetch(shard_id) if shard is None: raise _errors.ShardingError(SHARD_NOT_FOUND % (shard_id, )) #Both of the shard_mappings retrieved will be of the same sharding #type. Hence it is safe to use one of them to retireve the sharding type. if shard_mappings[0].type_name == "HASH": return HashShardingSpecification.fetch(shard_id), \ shard, shard_mappings, shard_mapping_defn else: return range_sharding_spec, shard, shard_mappings, shard_mapping_defn
def test_hash_remove(self): """Test the removal of hash shards. """ hash_sharding_specification_1 = HashShardingSpecification.fetch(1) hash_sharding_specification_2 = HashShardingSpecification.fetch(2) hash_sharding_specification_3 = HashShardingSpecification.fetch(3) hash_sharding_specification_4 = HashShardingSpecification.fetch(4) hash_sharding_specification_5 = HashShardingSpecification.fetch(5) hash_sharding_specification_1.remove() hash_sharding_specification_2.remove() hash_sharding_specification_3.remove() hash_sharding_specification_4.remove() hash_sharding_specification_5.remove() self.__shard_1.remove() self.__shard_2.remove() self.__shard_3.remove() self.__shard_4.remove() self.__shard_5.remove() for i in range(0, 10): hash_sharding_spec = HashShardingSpecification.lookup( i, self.__shard_mapping_id_1, "HASH") self.assertEqual(hash_sharding_spec, None)
def verify_and_fetch_shard(shard_id): """Find out if the shard_id exists and return the sharding specification for it. If it does not exist throw an exception. :param shard_id: The ID for the shard whose specification needs to be fetched. :return: The sharding specification class representing the shard ID. :raises: ShardingError if the shard ID is not found. """ #Here the underlying sharding specification might be a RANGE #or a HASH. The type of sharding specification is obtained from the #shard mapping. range_sharding_spec = RangeShardingSpecification.fetch(shard_id) if range_sharding_spec is None: raise _errors.ShardingError(SHARD_NOT_FOUND % (shard_id, )) #Fetch the shard mappings and use them to find the type of sharding #scheme. shard_mappings = ShardMapping.fetch_by_id( range_sharding_spec.shard_mapping_id) if shard_mappings is None: raise _errors.ShardingError(SHARD_MAPPING_NOT_FOUND % (range_sharding_spec.shard_mapping_id, )) #Fetch the shard mapping definition. There is only one shard mapping #definition associated with all of the shard mappings. shard_mapping_defn = ShardMapping.fetch_shard_mapping_defn( range_sharding_spec.shard_mapping_id) if shard_mapping_defn is None: raise _errors.ShardingError(SHARD_MAPPING_DEFN_NOT_FOUND % (range_sharding_spec.shard_mapping_id, )) shard = Shards.fetch(shard_id) if shard is None: raise _errors.ShardingError(SHARD_NOT_FOUND % (shard_id, )) #Both of the shard_mappings retrieved will be of the same sharding #type. Hence it is safe to use one of them to retireve the sharding type. if shard_mappings[0].type_name == "HASH": return HashShardingSpecification.fetch(shard_id), \ shard, shard_mappings, shard_mapping_defn else: return range_sharding_spec, shard, shard_mappings, shard_mapping_defn
def test_shard_split(self): split_cnt_1 = 0 split_cnt_2 = 0 shard_server_1 = None shard_server_2 = None expected_address_list_1 = \ [MySQLInstances().get_address(2), MySQLInstances().get_address(3)] expected_address_list_2 = \ [MySQLInstances().get_address(4), MySQLInstances().get_address(5)] status = self.proxy.sharding.split_shard("1", "GROUPID3") self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_prune_shard_tables_after_split).") for i in range(1, 100): status = self.proxy.sharding.lookup_servers("db1.t1", i, "LOCAL") self.assertEqual(status[0], True) self.assertEqual(status[1], "") obtained_server_list = status[2] obtained_uuid_list = [obtained_server_list[0][0], obtained_server_list[1][0]] obtained_address_list = [obtained_server_list[0][1], obtained_server_list[1][1]] try: self.assertEqual( set(expected_address_list_1), set(obtained_address_list) ) split_cnt_1 = split_cnt_1 + 1 if shard_server_1 is None: shard_server_1 = MySQLServer.fetch(obtained_uuid_list[0]) except AssertionError: self.assertEqual( set(expected_address_list_2), set(obtained_address_list) ) split_cnt_2 = split_cnt_2 + 1 if shard_server_2 is None: shard_server_2 = MySQLServer.fetch(obtained_uuid_list[0]) #Ensure that both the splits have been utilized. self.assertTrue(split_cnt_1 > 0) self.assertTrue(split_cnt_2 > 0) shard_server_1.connect() shard_server_2.connect() row_cnt_shard_1 = shard_server_1.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) row_cnt_shard_2 = shard_server_2.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch" : True} ) #Ensure that the split has happened, the number of values in #each shard should be less than the original. self.assertTrue(int(row_cnt_shard_1[0][0]) < 100) self.assertTrue(int(row_cnt_shard_2[0][0]) < 100) #Ensure tha two new shard_ids have been generated. hash_sharding_specifications = HashShardingSpecification.list(1) self.assertTrue(ShardingUtils.compare_hash_specifications( hash_sharding_specifications[1], HashShardingSpecification.fetch(2))) self.assertTrue(ShardingUtils.compare_hash_specifications( hash_sharding_specifications[0], HashShardingSpecification.fetch(3)))
def test_shard_split(self): split_cnt_1 = 0 split_cnt_2 = 0 shard_server_1 = None shard_server_2 = None expected_address_list_1 = \ [MySQLInstances().get_address(2), MySQLInstances().get_address(3)] expected_address_list_2 = \ [MySQLInstances().get_address(4), MySQLInstances().get_address(5)] status = self.proxy.sharding.split_shard("1", "GROUPID3") self.check_xmlrpc_command_result(status) for i in range(1, 100): status = self.proxy.sharding.lookup_servers("db1.t1", i, "LOCAL") obtained_uuid_list = [ info['server_uuid'] for info in self.check_xmlrpc_iter(status) ] obtained_address_list = [ info['address'] for info in self.check_xmlrpc_iter(status) ] try: self.assertEqual(set(expected_address_list_1), set(obtained_address_list)) split_cnt_1 = split_cnt_1 + 1 if shard_server_1 is None: shard_server_1 = fetch_test_server(obtained_uuid_list[0]) except AssertionError: self.assertEqual(set(expected_address_list_2), set(obtained_address_list)) split_cnt_2 = split_cnt_2 + 1 if shard_server_2 is None: shard_server_2 = fetch_test_server(obtained_uuid_list[0]) #Ensure that both the splits have been utilized. self.assertTrue(split_cnt_1 > 0) self.assertTrue(split_cnt_2 > 0) shard_server_1.connect() shard_server_2.connect() row_cnt_shard_1 = shard_server_1.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) row_cnt_shard_2 = shard_server_2.exec_stmt( "SELECT COUNT(*) FROM db1.t1", {"fetch": True}) #Ensure that the split has happened, the number of values in #each shard should be less than the original. self.assertTrue(int(row_cnt_shard_1[0][0]) < 100) self.assertTrue(int(row_cnt_shard_2[0][0]) < 100) #Ensure tha two new shard_ids have been generated. hash_sharding_specifications = HashShardingSpecification.list(1) self.assertTrue( ShardingUtils.compare_hash_specifications( hash_sharding_specifications[1], HashShardingSpecification.fetch(2))) self.assertTrue( ShardingUtils.compare_hash_specifications( hash_sharding_specifications[0], HashShardingSpecification.fetch(3)))