def test_hash_remove(self):
"""Test the removal of hash shards.
"""
hash_sharding_specification_1 = HashShardingSpecification.fetch(1)
hash_sharding_specification_2 = HashShardingSpecification.fetch(2)
hash_sharding_specification_3 = HashShardingSpecification.fetch(3)
hash_sharding_specification_4 = HashShardingSpecification.fetch(4)
hash_sharding_specification_5 = HashShardingSpecification.fetch(5)
hash_sharding_specification_1.remove()
hash_sharding_specification_2.remove()
hash_sharding_specification_3.remove()
hash_sharding_specification_4.remove()
hash_sharding_specification_5.remove()
self.__shard_1.remove()
self.__shard_2.remove()
self.__shard_3.remove()
self.__shard_4.remove()
self.__shard_5.remove()
for i in range(0, 10):
hash_sharding_spec = HashShardingSpecification.lookup(
i,
self.__shard_mapping_id_1,
"HASH"
)
self.assertEqual(hash_sharding_spec, None)
def test_fetch_sharding_scheme(self):
"""Test the fetch method of the HASH sharding scheme.
"""
hash_sharding_specification_1 = HashShardingSpecification.fetch(1)
hash_sharding_specification_2 = HashShardingSpecification.fetch(2)
hash_sharding_specification_3 = HashShardingSpecification.fetch(3)
hash_sharding_specification_4 = HashShardingSpecification.fetch(4)
hash_sharding_specification_5 = HashShardingSpecification.fetch(5)
hash_sharding_specifications = HashShardingSpecification.list(1)
#list does not return the hashing specifications in order of shard_id,
#hence a direct comparison is not posssible.
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_1, hash_sharding_specifications))
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_2, hash_sharding_specifications))
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_3, hash_sharding_specifications))
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_4, hash_sharding_specifications))
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_5, hash_sharding_specifications))
def test_fetch_sharding_scheme(self):
"""Test the fetch method of the HASH sharding scheme.
"""
hash_sharding_specification_1 = HashShardingSpecification.fetch(1)
hash_sharding_specification_2 = HashShardingSpecification.fetch(2)
hash_sharding_specification_3 = HashShardingSpecification.fetch(3)
hash_sharding_specification_4 = HashShardingSpecification.fetch(4)
hash_sharding_specification_5 = HashShardingSpecification.fetch(5)
hash_sharding_specifications = HashShardingSpecification.list(1)
#list does not return the hashing specifications in order of shard_id,
#hence a direct comparison is not posssible.
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_1,
hash_sharding_specifications
)
)
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_2,
hash_sharding_specifications
)
)
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_3,
hash_sharding_specifications
)
)
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_4,
hash_sharding_specifications
)
)
self.assertTrue(
self.hash_sharding_specification_in_list(
hash_sharding_specification_5,
hash_sharding_specifications
)
)
def verify_and_fetch_shard(shard_id):
"""Find out if the shard_id exists and return the sharding specification for
it. If it does not exist throw an exception.
:param shard_id: The ID for the shard whose specification needs to be
fetched.
:return: The sharding specification class representing the shard ID.
:raises: ShardingError if the shard ID is not found.
"""
#Here the underlying sharding specification might be a RANGE
#or a HASH. The type of sharding specification is obtained from the
#shard mapping.
range_sharding_spec = RangeShardingSpecification.fetch(shard_id)
if range_sharding_spec is None:
raise _errors.ShardingError(SHARD_NOT_FOUND % (shard_id, ))
#Fetch the shard mappings and use them to find the type of sharding
#scheme.
shard_mappings = ShardMapping.fetch_by_id(
range_sharding_spec.shard_mapping_id
)
if shard_mappings is None:
raise _errors.ShardingError(
SHARD_MAPPING_NOT_FOUND % (
range_sharding_spec.shard_mapping_id,
)
)
#Fetch the shard mapping definition. There is only one shard mapping
#definition associated with all of the shard mappings.
shard_mapping_defn = ShardMapping.fetch_shard_mapping_defn(
range_sharding_spec.shard_mapping_id
)
if shard_mapping_defn is None:
raise _errors.ShardingError(
SHARD_MAPPING_DEFN_NOT_FOUND % (
range_sharding_spec.shard_mapping_id,
)
)
shard = Shards.fetch(shard_id)
if shard is None:
raise _errors.ShardingError(SHARD_NOT_FOUND % (shard_id, ))
#Both of the shard_mappings retrieved will be of the same sharding
#type. Hence it is safe to use one of them to retireve the sharding type.
if shard_mappings[0].type_name == "HASH":
return HashShardingSpecification.fetch(shard_id), \
shard, shard_mappings, shard_mapping_defn
else:
return range_sharding_spec, shard, shard_mappings, shard_mapping_defn
def test_hash_remove(self):
"""Test the removal of hash shards.
"""
hash_sharding_specification_1 = HashShardingSpecification.fetch(1)
hash_sharding_specification_2 = HashShardingSpecification.fetch(2)
hash_sharding_specification_3 = HashShardingSpecification.fetch(3)
hash_sharding_specification_4 = HashShardingSpecification.fetch(4)
hash_sharding_specification_5 = HashShardingSpecification.fetch(5)
hash_sharding_specification_1.remove()
hash_sharding_specification_2.remove()
hash_sharding_specification_3.remove()
hash_sharding_specification_4.remove()
hash_sharding_specification_5.remove()
self.__shard_1.remove()
self.__shard_2.remove()
self.__shard_3.remove()
self.__shard_4.remove()
self.__shard_5.remove()
for i in range(0, 10):
hash_sharding_spec = HashShardingSpecification.lookup(
i, self.__shard_mapping_id_1, "HASH")
self.assertEqual(hash_sharding_spec, None)
def verify_and_fetch_shard(shard_id):
"""Find out if the shard_id exists and return the sharding specification for
it. If it does not exist throw an exception.
:param shard_id: The ID for the shard whose specification needs to be
fetched.
:return: The sharding specification class representing the shard ID.
:raises: ShardingError if the shard ID is not found.
"""
#Here the underlying sharding specification might be a RANGE
#or a HASH. The type of sharding specification is obtained from the
#shard mapping.
range_sharding_spec = RangeShardingSpecification.fetch(shard_id)
if range_sharding_spec is None:
raise _errors.ShardingError(SHARD_NOT_FOUND % (shard_id, ))
#Fetch the shard mappings and use them to find the type of sharding
#scheme.
shard_mappings = ShardMapping.fetch_by_id(
range_sharding_spec.shard_mapping_id)
if shard_mappings is None:
raise _errors.ShardingError(SHARD_MAPPING_NOT_FOUND %
(range_sharding_spec.shard_mapping_id, ))
#Fetch the shard mapping definition. There is only one shard mapping
#definition associated with all of the shard mappings.
shard_mapping_defn = ShardMapping.fetch_shard_mapping_defn(
range_sharding_spec.shard_mapping_id)
if shard_mapping_defn is None:
raise _errors.ShardingError(SHARD_MAPPING_DEFN_NOT_FOUND %
(range_sharding_spec.shard_mapping_id, ))
shard = Shards.fetch(shard_id)
if shard is None:
raise _errors.ShardingError(SHARD_NOT_FOUND % (shard_id, ))
#Both of the shard_mappings retrieved will be of the same sharding
#type. Hence it is safe to use one of them to retireve the sharding type.
if shard_mappings[0].type_name == "HASH":
return HashShardingSpecification.fetch(shard_id), \
shard, shard_mappings, shard_mapping_defn
else:
return range_sharding_spec, shard, shard_mappings, shard_mapping_defn
def test_shard_split(self):
split_cnt_1 = 0
split_cnt_2 = 0
shard_server_1 = None
shard_server_2 = None
expected_address_list_1 = \
[MySQLInstances().get_address(2), MySQLInstances().get_address(3)]
expected_address_list_2 = \
[MySQLInstances().get_address(4), MySQLInstances().get_address(5)]
status = self.proxy.sharding.split_shard("1", "GROUPID3")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
for i in range(1, 100):
status = self.proxy.sharding.lookup_servers("db1.t1", i, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
obtained_uuid_list = [obtained_server_list[0][0],
obtained_server_list[1][0]]
obtained_address_list = [obtained_server_list[0][1],
obtained_server_list[1][1]]
try:
self.assertEqual(
set(expected_address_list_1), set(obtained_address_list)
)
split_cnt_1 = split_cnt_1 + 1
if shard_server_1 is None:
shard_server_1 = MySQLServer.fetch(obtained_uuid_list[0])
except AssertionError:
self.assertEqual(
set(expected_address_list_2), set(obtained_address_list)
)
split_cnt_2 = split_cnt_2 + 1
if shard_server_2 is None:
shard_server_2 = MySQLServer.fetch(obtained_uuid_list[0])
#Ensure that both the splits have been utilized.
self.assertTrue(split_cnt_1 > 0)
self.assertTrue(split_cnt_2 > 0)
shard_server_1.connect()
shard_server_2.connect()
row_cnt_shard_1 = shard_server_1.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
row_cnt_shard_2 = shard_server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1",
{"fetch" : True}
)
#Ensure that the split has happened, the number of values in
#each shard should be less than the original.
self.assertTrue(int(row_cnt_shard_1[0][0]) < 100)
self.assertTrue(int(row_cnt_shard_2[0][0]) < 100)
#Ensure tha two new shard_ids have been generated.
hash_sharding_specifications = HashShardingSpecification.list(1)
self.assertTrue(ShardingUtils.compare_hash_specifications(
hash_sharding_specifications[1],
HashShardingSpecification.fetch(2)))
self.assertTrue(ShardingUtils.compare_hash_specifications(
hash_sharding_specifications[0],
HashShardingSpecification.fetch(3)))
def test_shard_split(self):
split_cnt_1 = 0
split_cnt_2 = 0
shard_server_1 = None
shard_server_2 = None
expected_address_list_1 = \
[MySQLInstances().get_address(2), MySQLInstances().get_address(3)]
expected_address_list_2 = \
[MySQLInstances().get_address(4), MySQLInstances().get_address(5)]
status = self.proxy.sharding.split_shard("1", "GROUPID3")
self.check_xmlrpc_command_result(status)
for i in range(1, 100):
status = self.proxy.sharding.lookup_servers("db1.t1", i, "LOCAL")
obtained_uuid_list = [
info['server_uuid'] for info in self.check_xmlrpc_iter(status)
]
obtained_address_list = [
info['address'] for info in self.check_xmlrpc_iter(status)
]
try:
self.assertEqual(set(expected_address_list_1),
set(obtained_address_list))
split_cnt_1 = split_cnt_1 + 1
if shard_server_1 is None:
shard_server_1 = fetch_test_server(obtained_uuid_list[0])
except AssertionError:
self.assertEqual(set(expected_address_list_2),
set(obtained_address_list))
split_cnt_2 = split_cnt_2 + 1
if shard_server_2 is None:
shard_server_2 = fetch_test_server(obtained_uuid_list[0])
#Ensure that both the splits have been utilized.
self.assertTrue(split_cnt_1 > 0)
self.assertTrue(split_cnt_2 > 0)
shard_server_1.connect()
shard_server_2.connect()
row_cnt_shard_1 = shard_server_1.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
row_cnt_shard_2 = shard_server_2.exec_stmt(
"SELECT COUNT(*) FROM db1.t1", {"fetch": True})
#Ensure that the split has happened, the number of values in
#each shard should be less than the original.
self.assertTrue(int(row_cnt_shard_1[0][0]) < 100)
self.assertTrue(int(row_cnt_shard_2[0][0]) < 100)
#Ensure tha two new shard_ids have been generated.
hash_sharding_specifications = HashShardingSpecification.list(1)
self.assertTrue(
ShardingUtils.compare_hash_specifications(
hash_sharding_specifications[1],
HashShardingSpecification.fetch(2)))
self.assertTrue(
ShardingUtils.compare_hash_specifications(
hash_sharding_specifications[0],
HashShardingSpecification.fetch(3)))
