def test_fetch_shard_mapping(self):
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_2 = ShardMapping.fetch("db2.t2")
shard_mapping_3 = ShardMapping.fetch("db3.t3")
shard_mapping_4 = ShardMapping.fetch("db4.t4")
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_1, shard_mapping_1))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_2, shard_mapping_2))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_3, shard_mapping_3))
self.assertTrue(ShardingUtils.compare_shard_mapping
(self.__shard_mapping_4, shard_mapping_4))
def _prune_shard_tables(table_name, prune_limit):
"""Delete the data from the copied data directories based on the
sharding configuration uploaded in the sharding tables of the state
store. The basic logic consists of
a) Querying the sharding scheme name corresponding to the sharding table
b) Querying the sharding key range using the sharding scheme name.
c) Deleting the sharding keys that fall outside the range for a given
server.
:param table_name: The table_name who's shards need to be pruned.
:param prune_limit: The number of DELETEs that should be
done in one batch.
"""
shard_mapping = ShardMapping.fetch(table_name)
try:
SHARDING_SPECIFICATION_HANDLER[shard_mapping.type_name].\
delete_from_shard_db(table_name, shard_mapping.type_name, prune_limit)
except _errors.DatabaseError as error:
if error.errno == ER_NO_SUCH_TABLE:
#Error happens because the actual tables are not present in the
#server. We will ignore this.
pass
else:
raise error
def _lookup_shard_mapping(table_name):
"""Fetch the shard specification mapping for the given table
:param table_name: The name of the table for which the sharding
specification is being queried.
:return: A dictionary that contains the shard mapping information for
the given table.
"""
shard_mapping = ShardMapping.fetch(table_name)
if shard_mapping is not None:
return [{"mapping_id":shard_mapping.shard_mapping_id,
"table_name":shard_mapping.table_name,
"column_name":shard_mapping.column_name,
"type_name":shard_mapping.type_name,
"global_group":shard_mapping.global_group}]
else:
#We return an empty shard mapping because if an Error is thrown
#it would cause the executor to rollback which is an unnecessary
#action. It is enough if we inform the user that the lookup returned
#nothing.
return [{"mapping_id":"",
"table_name":"",
"column_name":"",
"type_name":"",
"global_group":""}]
def _prune_shard_tables(table_name, prune_limit):
"""Delete the data from the copied data directories based on the
sharding configuration uploaded in the sharding tables of the state
store. The basic logic consists of
a) Querying the sharding scheme name corresponding to the sharding table
b) Querying the sharding key range using the sharding scheme name.
c) Deleting the sharding keys that fall outside the range for a given
server.
:param table_name: The table_name who's shards need to be pruned.
:param prune_limit: The number of DELETEs that should be
done in one batch.
"""
shard_mapping = ShardMapping.fetch(table_name)
try:
SHARDING_SPECIFICATION_HANDLER[shard_mapping.type_name].\
delete_from_shard_db(table_name, shard_mapping.type_name, prune_limit)
except _errors.DatabaseError as error:
if error.errno == ER_NO_SUCH_TABLE:
#Error happens because the actual tables are not present in the
#server. We will ignore this.
pass
else:
raise error
def _lookup_shard_mapping(table_name):
"""Fetch the shard specification mapping for the given table
:param table_name: The name of the table for which the sharding
specification is being queried.
:return: A dictionary that contains the shard mapping information for
the given table.
"""
shard_mapping = ShardMapping.fetch(table_name)
if shard_mapping is not None:
return [{
"mapping_id": shard_mapping.shard_mapping_id,
"table_name": shard_mapping.table_name,
"column_name": shard_mapping.column_name,
"type_name": shard_mapping.type_name,
"global_group": shard_mapping.global_group
}]
else:
#We return an empty shard mapping because if an Error is thrown
#it would cause the executor to rollback which is an unnecessary
#action. It is enough if we inform the user that the lookup returned
#nothing.
return [{
"mapping_id": "",
"table_name": "",
"column_name": "",
"type_name": "",
"global_group": ""
}]
def test_fetch_shard_mapping(self):
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_2 = ShardMapping.fetch("db2.t2")
shard_mapping_3 = ShardMapping.fetch("db3.t3")
shard_mapping_4 = ShardMapping.fetch("db4.t4")
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_1,
shard_mapping_1))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_2,
shard_mapping_2))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_3,
shard_mapping_3))
self.assertTrue(
ShardingUtils.compare_shard_mapping(self.__shard_mapping_4,
shard_mapping_4))
def _lookup(lookup_arg, key, hint):
"""Given a table name and a key return the servers of the Group where the
shard of this table can be found
:param lookup_arg: table name for "LOCAL" lookups
Shard Mapping ID for "GLOBAL" lookups.
:param key: The key value that needs to be looked up
:param hint: A hint indicates if the query is LOCAL or GLOBAL
:return: The servers of the Group that contains the range in which the
key belongs.
"""
VALID_HINTS = ('LOCAL', 'GLOBAL')
hint = hint.upper()
if hint not in VALID_HINTS:
raise _errors.ShardingError(INVALID_SHARDING_HINT)
group = None
#Perform the lookup for the group contaning the lookup data.
if hint == "GLOBAL":
#Fetch the shard mapping object. In the case of GLOBAL lookups
#the shard mapping ID is passed directly. In the case of "LOCAL"
#lookups it is the table name that is passed.
shard_mapping = ShardMapping.fetch_by_id(lookup_arg)
if shard_mapping is None:
raise _errors.ShardingError(
SHARD_MAPPING_NOT_FOUND % (lookup_arg, )
)
#GLOBAL lookups. There can be only one global group, hence using
#shard_mapping[0] is safe.
group = Group.fetch(shard_mapping[0].global_group)
else:
shard_mapping = ShardMapping.fetch(lookup_arg)
if shard_mapping is None:
raise _errors.ShardingError(TABLE_NAME_NOT_FOUND % (lookup_arg, ))
sharding_specification =\
SHARDING_SPECIFICATION_HANDLER[shard_mapping.type_name].\
lookup(key, shard_mapping.shard_mapping_id, shard_mapping.type_name)
if sharding_specification is None:
raise _errors.ShardingError(INVALID_SHARDING_KEY % (key, ))
shard = Shards.fetch(str(sharding_specification.shard_id))
if shard.state == "DISABLED":
raise _errors.ShardingError(SHARD_NOT_ENABLED)
#group cannot be None since there is a foreign key on the group_id.
#An exception will be thrown nevertheless.
group = Group.fetch(shard.group_id)
if group is None:
raise _errors.ShardingError(SHARD_LOCATION_NOT_FOUND)
ret = []
#An empty list will be returned if the registered group has not
#servers.
for server in group.servers():
ret.append([str(server.uuid), server.address,
group.master == server.uuid])
return ret
def _remove_shard_mapping(table_name):
"""Remove the shard mapping for the given table.
:param table_name: The name of the table for which the shard mapping
needs to be removed.
:return: True if the remove succeeded
False if the query failed
:raises: ShardingError if the table name is not found.
"""
shard_mapping = ShardMapping.fetch(table_name)
if shard_mapping is None:
raise _errors.ShardingError(TABLE_NAME_NOT_FOUND % (table_name, ))
shard_mapping.remove()
def _remove_shard_mapping(table_name):
"""Remove the shard mapping for the given table.
:param table_name: The name of the table for which the shard mapping
needs to be removed.
:return: True if the remove succeeded
False if the query failed
:raises: ShardingError if the table name is not found.
"""
shard_mapping = ShardMapping.fetch(table_name)
if shard_mapping is None:
raise _errors.ShardingError(TABLE_NAME_NOT_FOUND % (table_name, ))
shard_mapping.remove()
def _prune_shard_tables(table_name):
"""Delete the data from the copied data directories based on the
sharding configuration uploaded in the sharding tables of the state
store. The basic logic consists of
a) Querying the sharding scheme name corresponding to the sharding table
b) Querying the sharding key range using the sharding scheme name.
c) Deleting the sharding keys that fall outside the range for a given
server.
:param table_name: The table_name who's shards need to be pruned.
"""
shard_mapping = ShardMapping.fetch(table_name)
SHARDING_SPECIFICATION_HANDLER[shard_mapping.type_name].\
delete_from_shard_db(table_name, shard_mapping.type_name)
def _lookup(lookup_arg, key, hint):
"""Given a table name and a key return the servers of the Group where the
shard of this table can be found
:param lookup_arg: table name for "LOCAL" lookups
Shard Mapping ID for "GLOBAL" lookups.
:param key: The key value that needs to be looked up
:param hint: A hint indicates if the query is LOCAL or GLOBAL
:return: The servers of the Group that contains the range in which the
key belongs.
"""
VALID_HINTS = ('LOCAL', 'GLOBAL')
hint = hint.upper()
if hint not in VALID_HINTS:
raise _errors.ShardingError(INVALID_SHARDING_HINT)
group = None
#Perform the lookup for the group contaning the lookup data.
if hint == "GLOBAL":
#Fetch the shard mapping object. In the case of GLOBAL lookups
#the shard mapping ID is passed directly. In the case of "LOCAL"
#lookups it is the table name that is passed.
shard_mapping = ShardMapping.fetch_by_id(lookup_arg)
if shard_mapping is None:
raise _errors.ShardingError(SHARD_MAPPING_NOT_FOUND %
(lookup_arg, ))
#GLOBAL lookups. There can be only one global group, hence using
#shard_mapping[0] is safe.
group_id = shard_mapping[0].global_group
else:
shard_mapping = ShardMapping.fetch(lookup_arg)
if shard_mapping is None:
raise _errors.ShardingError(TABLE_NAME_NOT_FOUND % (lookup_arg, ))
sharding_specification = \
SHARDING_SPECIFICATION_HANDLER[shard_mapping.type_name].\
lookup(key, shard_mapping.shard_mapping_id, shard_mapping.type_name)
if sharding_specification is None:
raise _errors.ShardingError(INVALID_SHARDING_KEY % (key, ))
shard = Shards.fetch(str(sharding_specification.shard_id))
if shard.state == "DISABLED":
raise _errors.ShardingError(SHARD_NOT_ENABLED)
#group cannot be None since there is a foreign key on the group_id.
#An exception will be thrown nevertheless.
group_id = shard.group_id
return ServerLookups().execute(group_id=group_id)
def test_shard_mapping_remove(self):
self.__range_sharding_specification_1.remove()
self.__range_sharding_specification_2.remove()
self.__range_sharding_specification_3.remove()
self.__range_sharding_specification_4.remove()
self.__range_sharding_specification_5.remove()
self.__range_sharding_specification_6.remove()
self.__range_sharding_specification_7.remove()
self.__range_sharding_specification_8.remove()
self.__range_sharding_specification_9.remove()
self.__range_sharding_specification_10.remove()
self.__shard_id_1.remove()
self.__shard_id_2.remove()
self.__shard_id_3.remove()
self.__shard_id_4.remove()
self.__shard_id_5.remove()
self.__shard_id_6.remove()
self.__shard_id_7.remove()
self.__shard_id_8.remove()
self.__shard_id_9.remove()
self.__shard_id_10.remove()
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_1.remove()
def test_shard_mapping_remove(self):
self.__range_sharding_specification_1.remove()
self.__range_sharding_specification_2.remove()
self.__range_sharding_specification_3.remove()
self.__range_sharding_specification_4.remove()
self.__range_sharding_specification_5.remove()
self.__range_sharding_specification_6.remove()
self.__range_sharding_specification_7.remove()
self.__range_sharding_specification_8.remove()
self.__range_sharding_specification_9.remove()
self.__range_sharding_specification_10.remove()
self.__shard_id_1.remove()
self.__shard_id_2.remove()
self.__shard_id_3.remove()
self.__shard_id_4.remove()
self.__shard_id_5.remove()
self.__shard_id_6.remove()
self.__shard_id_7.remove()
self.__shard_id_8.remove()
self.__shard_id_9.remove()
self.__shard_id_10.remove()
shard_mapping_1 = ShardMapping.fetch("db1.t1")
shard_mapping_1.remove()
