def test_fetch_sharding_scheme(self):
range_sharding_specifications = RangeShardingSpecification.list(1)
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[0],
self.__range_sharding_specification_1))
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[1],
self.__range_sharding_specification_2))
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[2],
self.__range_sharding_specification_3))
def test_fetch_sharding_scheme(self):
range_sharding_specifications = RangeShardingSpecification.list(1)
self.assertTrue(
ShardingUtils.compare_range_specifications(
range_sharding_specifications[0],
self.__range_sharding_specification_1))
self.assertTrue(
ShardingUtils.compare_range_specifications(
range_sharding_specifications[1],
self.__range_sharding_specification_2))
self.assertTrue(
ShardingUtils.compare_range_specifications(
range_sharding_specifications[2],
self.__range_sharding_specification_3))
def _add_shard(shard_mapping_id, groupid_lb_list, state, update_only=False):
"""Add the RANGE shard specification. This represents a single instance
of a shard specification that maps a key RANGE to a server.
:param shard_mapping_id: The unique identification for a shard mapping.
:param groupid_lb_list: The list of group_id, lower_bounds pairs in the
format, group_id/lower_bound, group_id/lower_bound... .
:param state: Indicates whether a given shard is ENABLED or DISABLED
:param update_only: Only update the state store and skip adding range checks.
:return: True if the add succeeded.
False otherwise.
:raises: ShardingError If the group on which the shard is being
created does not exist,
If the shard_mapping_id is not found,
If adding the shard definition fails,
If the state of the shard is an invalid
value,
If the range definition is invalid.
"""
shard_mapping = ShardMapping.fetch_shard_mapping_defn(shard_mapping_id)
if shard_mapping is None:
raise _errors.ShardingError(SHARD_MAPPING_NOT_FOUND % \
(shard_mapping_id, ))
schema_type = shard_mapping[1]
if len(RangeShardingSpecification.list(shard_mapping_id)) != 0:
raise _errors.ShardingError(SHARDS_ALREADY_EXIST)
group_id_list, lower_bound_list = \
_utils.get_group_lower_bound_list(groupid_lb_list)
if (len(group_id_list) != len(lower_bound_list)) and\
schema_type == "RANGE":
raise _errors.ShardingError(LOWER_BOUND_GROUP_ID_COUNT_MISMATCH)
if len(lower_bound_list) != 0 and schema_type == "HASH":
raise _errors.ShardingError(LOWER_BOUND_AUTO_GENERATED)
if schema_type in Shards.VALID_RANGE_SHARDING_TYPES:
for lower_bound in lower_bound_list:
if not SHARDING_DATATYPE_HANDLER[schema_type].\
is_valid_lower_bound(lower_bound):
raise _errors.ShardingError(
INVALID_LOWER_BOUND_VALUE % (lower_bound, ))
state = state.upper()
if state not in Shards.VALID_SHARD_STATES:
raise _errors.ShardingError(INVALID_SHARD_STATE % (state, ))
for index, group_id in enumerate(group_id_list):
shard = Shards.add(group_id, state)
shard_id = shard.shard_id
if schema_type == "HASH":
HashShardingSpecification.add(
shard_mapping_id,
shard_id
)
_LOGGER.debug(
"Added Shard (map id = %s, id = %s).",
shard_mapping_id,
shard_id
)
else:
range_sharding_specification = \
SHARDING_SPECIFICATION_HANDLER[schema_type].add(
shard_mapping_id,
lower_bound_list[index],
shard_id
)
_LOGGER.debug(
"Added Shard (map id = %s, lower bound = %s, id = %s).",
range_sharding_specification.shard_mapping_id,
range_sharding_specification.lower_bound,
range_sharding_specification.shard_id
)
if not update_only:
#If the shard is added in a DISABLED state do not setup replication
#with the primary of the global group. Basically setup replication only
#if the shard is ENABLED.
if state == "ENABLED":
_setup_shard_group_replication(shard_id)
if not update_only:
#Add the shard limits into the metadata present in each of the shards.
_events.trigger_within_procedure(
ADD_SHARD_RANGE_CHECK,
shard_mapping_id,
schema_type
)
def test_shard_split(self):
status = self.proxy.sharding.split_shard("1", "GROUPID3", "600")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_prune_shard_tables_after_split).")
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
for idx in range(0, 2):
server_uuid = obtained_server_list[idx][0]
shard_server = MySQLServer.fetch(server_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT NAME FROM db1.t1",
{"fetch" : True})
self.assertEqual(len(rows), 3)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
self.assertEqual(rows[2][0], 'TEST 3')
status = self.proxy.sharding.lookup_servers("db1.t1", 800, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
for idx in range(0, 2):
server_uuid = obtained_server_list[idx][0]
shard_server = MySQLServer.fetch(server_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT NAME FROM db1.t1",
{"fetch" : True})
self.assertEqual(len(rows), 4)
self.assertEqual(rows[0][0], 'TEST 4')
self.assertEqual(rows[1][0], 'TEST 5')
self.assertEqual(rows[2][0], 'TEST 6')
self.assertEqual(rows[3][0], 'TEST 7')
status = self.proxy.sharding.lookup_servers("1", 500, "GLOBAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
for idx in range(0, 2):
if obtained_server_list[idx][2]:
global_master_uuid = obtained_server_list[idx][0]
break
global_master = MySQLServer.fetch(global_master_uuid)
global_master.connect()
global_master.exec_stmt("DROP DATABASE IF EXISTS global_db")
global_master.exec_stmt("CREATE DATABASE global_db")
global_master.exec_stmt("CREATE TABLE global_db.global_table"
"(userID INT, name VARCHAR(30))")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(101, 'TEST 1')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(202, 'TEST 2')")
status = self.proxy.group.promote("GROUPID1")
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_change_to_candidate).")
sleep(5)
status = self.proxy.sharding.lookup_servers("1", 500, "GLOBAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
for idx in range(0, 2):
if obtained_server_list[idx][2]:
global_master_uuid = obtained_server_list[idx][0]
break
global_master = MySQLServer.fetch(global_master_uuid)
global_master.connect()
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(303, 'TEST 3')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(404, 'TEST 4')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(505, 'TEST 5')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(606, 'TEST 6')")
sleep(5)
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
self.assertEqual(status[0], True)
self.assertEqual(status[1], "")
obtained_server_list = status[2]
for idx in range(0, 2):
if obtained_server_list[idx][2]:
shard_uuid = obtained_server_list[idx][0]
shard_server = MySQLServer.fetch(shard_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT NAME FROM global_db.global_table", {"fetch" : True}
)
self.assertEqual(len(rows), 6)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
self.assertEqual(rows[2][0], 'TEST 3')
self.assertEqual(rows[3][0], 'TEST 4')
self.assertEqual(rows[4][0], 'TEST 5')
self.assertEqual(rows[5][0], 'TEST 6')
#Ensure tha two new shard_ids have been generated.
range_sharding_specifications = RangeShardingSpecification.list(1)
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[0],
RangeShardingSpecification.fetch(2)))
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[1],
RangeShardingSpecification.fetch(3)))
def test_shard_split(self):
status = self.proxy.sharding.split_shard("1", "GROUPID3", "600")
self.check_xmlrpc_command_result(status)
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
for info in self.check_xmlrpc_iter(status):
server_uuid = info['server_uuid']
shard_server = fetch_test_server(server_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT NAME FROM db1.t1",
{"fetch" : True})
self.assertEqual(len(rows), 3)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
self.assertEqual(rows[2][0], 'TEST 3')
status = self.proxy.sharding.lookup_servers("db1.t1", 800, "LOCAL")
for info in self.check_xmlrpc_iter(status):
server_uuid = info['server_uuid']
shard_server = fetch_test_server(server_uuid)
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT NAME FROM db1.t1",
{"fetch" : True})
self.assertEqual(len(rows), 4)
self.assertEqual(rows[0][0], 'TEST 4')
self.assertEqual(rows[1][0], 'TEST 5')
self.assertEqual(rows[2][0], 'TEST 6')
self.assertEqual(rows[3][0], 'TEST 7')
status = self.proxy.sharding.lookup_servers("1", 500, "GLOBAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
global_master = fetch_test_server(info['server_uuid'])
global_master.connect()
global_master.exec_stmt("DROP DATABASE IF EXISTS global_db")
global_master.exec_stmt("CREATE DATABASE global_db")
global_master.exec_stmt("CREATE TABLE global_db.global_table"
"(userID INT, name VARCHAR(30))")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(101, 'TEST 1')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(202, 'TEST 2')")
status = self.proxy.group.promote("GROUPID1")
self.check_xmlrpc_command_result(status)
sleep(5)
status = self.proxy.sharding.lookup_servers("1", 500, "GLOBAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
global_master = fetch_test_server(info['server_uuid'])
global_master.connect()
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(303, 'TEST 3')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(404, 'TEST 4')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(505, 'TEST 5')")
global_master.exec_stmt("INSERT INTO global_db.global_table "
"VALUES(606, 'TEST 6')")
sleep(5)
status = self.proxy.sharding.lookup_servers("db1.t1", 500, "LOCAL")
for info in self.check_xmlrpc_iter(status):
if info['status'] == MySQLServer.PRIMARY:
shard_server = fetch_test_server(info['server_uuid'])
shard_server.connect()
rows = shard_server.exec_stmt(
"SELECT NAME FROM global_db.global_table", {"fetch" : True}
)
self.assertEqual(len(rows), 6)
self.assertEqual(rows[0][0], 'TEST 1')
self.assertEqual(rows[1][0], 'TEST 2')
self.assertEqual(rows[2][0], 'TEST 3')
self.assertEqual(rows[3][0], 'TEST 4')
self.assertEqual(rows[4][0], 'TEST 5')
self.assertEqual(rows[5][0], 'TEST 6')
#Ensure tha two new shard_ids have been generated.
range_sharding_specifications = RangeShardingSpecification.list(1)
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[0],
RangeShardingSpecification.fetch(2)))
self.assertTrue(ShardingUtils.compare_range_specifications(
range_sharding_specifications[1],
RangeShardingSpecification.fetch(3)))
def _add_shard(shard_mapping_id, groupid_lb_list, state):
"""Add the RANGE shard specification. This represents a single instance
of a shard specification that maps a key RANGE to a server.
:param shard_mapping_id: The unique identification for a shard mapping.
:param groupid_lb_list: The list of group_id, lower_bounds pairs in the
format, group_id/lower_bound, group_id/lower_bound... .
:param state: Indicates whether a given shard is ENABLED or DISABLED
:return: True if the add succeeded.
False otherwise.
:raises: ShardingError If the group on which the shard is being
created does not exist,
If the shard_mapping_id is not found,
If adding the shard definition fails,
If the state of the shard is an invalid
value,
If the range definition is invalid.
"""
shard_mapping = ShardMapping.fetch_shard_mapping_defn(shard_mapping_id)
if shard_mapping is None:
raise _errors.ShardingError(SHARD_MAPPING_NOT_FOUND % \
(shard_mapping_id, ))
schema_type = shard_mapping[1]
if len(RangeShardingSpecification.list(shard_mapping_id)) != 0:
raise _errors.ShardingError(SHARDS_ALREADY_EXIST)
group_id_list, lower_bound_list = \
_utils.get_group_lower_bound_list(groupid_lb_list)
if (len(group_id_list) != len(lower_bound_list)) and\
schema_type == "RANGE":
raise _errors.ShardingError(LOWER_BOUND_GROUP_ID_COUNT_MISMATCH)
if len(lower_bound_list) != 0 and schema_type == "HASH":
raise _errors.ShardingError(LOWER_BOUND_AUTO_GENERATED)
if schema_type in Shards.VALID_RANGE_SHARDING_TYPES:
for lower_bound in lower_bound_list:
if not SHARDING_DATATYPE_HANDLER[schema_type].\
is_valid_lower_bound(lower_bound):
raise _errors.ShardingError(INVALID_LOWER_BOUND_VALUE %
(lower_bound, ))
state = state.upper()
if state not in Shards.VALID_SHARD_STATES:
raise _errors.ShardingError(INVALID_SHARD_STATE % (state, ))
for index, group_id in enumerate(group_id_list):
shard = Shards.add(group_id, state)
shard_id = shard.shard_id
if schema_type == "HASH":
HashShardingSpecification.add(shard_mapping_id, shard_id)
_LOGGER.debug("Added Shard (map id = %s, id = %s).",
shard_mapping_id, shard_id)
else:
range_sharding_specification = \
SHARDING_SPECIFICATION_HANDLER[schema_type].add(
shard_mapping_id,
lower_bound_list[index],
shard_id
)
_LOGGER.debug(
"Added Shard (map id = %s, lower bound = %s, id = %s).",
range_sharding_specification.shard_mapping_id,
range_sharding_specification.lower_bound,
range_sharding_specification.shard_id)
#If the shard is added in a DISABLED state do not setup replication
#with the primary of the global group. Basically setup replication only
#if the shard is ENABLED.
if state == "ENABLED":
_setup_shard_group_replication(shard_id)