def _add_shard(shard_mapping_id, groupid_lb_list, state, update_only=False): """Add the RANGE shard specification. This represents a single instance of a shard specification that maps a key RANGE to a server. :param shard_mapping_id: The unique identification for a shard mapping. :param groupid_lb_list: The list of group_id, lower_bounds pairs in the format, group_id/lower_bound, group_id/lower_bound... . :param state: Indicates whether a given shard is ENABLED or DISABLED :param update_only: Only update the state store and skip adding range checks. :return: True if the add succeeded. False otherwise. :raises: ShardingError If the group on which the shard is being created does not exist, If the shard_mapping_id is not found, If adding the shard definition fails, If the state of the shard is an invalid value, If the range definition is invalid. """ shard_mapping = ShardMapping.fetch_shard_mapping_defn(shard_mapping_id) if shard_mapping is None: raise _errors.ShardingError(SHARD_MAPPING_NOT_FOUND % \ (shard_mapping_id, )) schema_type = shard_mapping[1] if len(RangeShardingSpecification.list(shard_mapping_id)) != 0: raise _errors.ShardingError(SHARDS_ALREADY_EXIST) group_id_list, lower_bound_list = \ _utils.get_group_lower_bound_list(groupid_lb_list) if (len(group_id_list) != len(lower_bound_list)) and\ schema_type == "RANGE": raise _errors.ShardingError(LOWER_BOUND_GROUP_ID_COUNT_MISMATCH) if len(lower_bound_list) != 0 and schema_type == "HASH": raise _errors.ShardingError(LOWER_BOUND_AUTO_GENERATED) if schema_type in Shards.VALID_RANGE_SHARDING_TYPES: for lower_bound in lower_bound_list: if not SHARDING_DATATYPE_HANDLER[schema_type].\ is_valid_lower_bound(lower_bound): raise _errors.ShardingError( INVALID_LOWER_BOUND_VALUE % (lower_bound, )) state = state.upper() if state not in Shards.VALID_SHARD_STATES: raise _errors.ShardingError(INVALID_SHARD_STATE % (state, )) for index, group_id in enumerate(group_id_list): shard = Shards.add(group_id, state) shard_id = shard.shard_id if schema_type == "HASH": HashShardingSpecification.add( shard_mapping_id, shard_id ) _LOGGER.debug( "Added Shard (map id = %s, id = %s).", shard_mapping_id, shard_id ) else: range_sharding_specification = \ SHARDING_SPECIFICATION_HANDLER[schema_type].add( shard_mapping_id, lower_bound_list[index], shard_id ) _LOGGER.debug( "Added Shard (map id = %s, lower bound = %s, id = %s).", range_sharding_specification.shard_mapping_id, range_sharding_specification.lower_bound, range_sharding_specification.shard_id ) if not update_only: #If the shard is added in a DISABLED state do not setup replication #with the primary of the global group. Basically setup replication only #if the shard is ENABLED. if state == "ENABLED": _setup_shard_group_replication(shard_id) if not update_only: #Add the shard limits into the metadata present in each of the shards. _events.trigger_within_procedure( ADD_SHARD_RANGE_CHECK, shard_mapping_id, schema_type )
def _add_shard(shard_mapping_id, groupid_lb_list, state): """Add the RANGE shard specification. This represents a single instance of a shard specification that maps a key RANGE to a server. :param shard_mapping_id: The unique identification for a shard mapping. :param groupid_lb_list: The list of group_id, lower_bounds pairs in the format, group_id/lower_bound, group_id/lower_bound... . :param state: Indicates whether a given shard is ENABLED or DISABLED :return: True if the add succeeded. False otherwise. :raises: ShardingError If the group on which the shard is being created does not exist, If the shard_mapping_id is not found, If adding the shard definition fails, If the state of the shard is an invalid value, If the range definition is invalid. """ shard_mapping = ShardMapping.fetch_shard_mapping_defn(shard_mapping_id) if shard_mapping is None: raise _errors.ShardingError(SHARD_MAPPING_NOT_FOUND % \ (shard_mapping_id, )) schema_type = shard_mapping[1] if len(RangeShardingSpecification.list(shard_mapping_id)) != 0: raise _errors.ShardingError(SHARDS_ALREADY_EXIST) group_id_list, lower_bound_list = \ _utils.get_group_lower_bound_list(groupid_lb_list) if (len(group_id_list) != len(lower_bound_list)) and\ schema_type == "RANGE": raise _errors.ShardingError(LOWER_BOUND_GROUP_ID_COUNT_MISMATCH) if len(lower_bound_list) != 0 and schema_type == "HASH": raise _errors.ShardingError(LOWER_BOUND_AUTO_GENERATED) if schema_type in Shards.VALID_RANGE_SHARDING_TYPES: for lower_bound in lower_bound_list: if not SHARDING_DATATYPE_HANDLER[schema_type].\ is_valid_lower_bound(lower_bound): raise _errors.ShardingError(INVALID_LOWER_BOUND_VALUE % (lower_bound, )) state = state.upper() if state not in Shards.VALID_SHARD_STATES: raise _errors.ShardingError(INVALID_SHARD_STATE % (state, )) for index, group_id in enumerate(group_id_list): shard = Shards.add(group_id, state) shard_id = shard.shard_id if schema_type == "HASH": HashShardingSpecification.add(shard_mapping_id, shard_id) _LOGGER.debug("Added Shard (map id = %s, id = %s).", shard_mapping_id, shard_id) else: range_sharding_specification = \ SHARDING_SPECIFICATION_HANDLER[schema_type].add( shard_mapping_id, lower_bound_list[index], shard_id ) _LOGGER.debug( "Added Shard (map id = %s, lower bound = %s, id = %s).", range_sharding_specification.shard_mapping_id, range_sharding_specification.lower_bound, range_sharding_specification.shard_id) #If the shard is added in a DISABLED state do not setup replication #with the primary of the global group. Basically setup replication only #if the shard is ENABLED. if state == "ENABLED": _setup_shard_group_replication(shard_id)
def setUp(self): self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid" : _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(0), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid" : _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(1), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid" : _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(2), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add( self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add( self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid" : _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(3), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add( self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid" : _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(4), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add( self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid" : _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address" : MySQLInstances().get_address(5), "user" : MySQLInstances().user, "passwd" : MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__server_6.connect() self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_6.exec_stmt("CREATE DATABASE db1") self.__server_6.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add( self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn() self.assertEquals( self.__shard_mapping_list, []) self.__shard_mapping_id_1 = ShardMapping.define("HASH", "GROUPID1") self.__shard_mapping_1 = ShardMapping.add( self.__shard_mapping_id_1, "db1.t1", "userID" ) self.__shard_1 = Shards.add("GROUPID2") self.__shard_2 = Shards.add("GROUPID3") self.__shard_3 = Shards.add("GROUPID4") self.__shard_4 = Shards.add("GROUPID5") self.__shard_5 = Shards.add("GROUPID6") self.__hash_sharding_specification_1 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_1.shard_id ) self.__hash_sharding_specification_2 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_2.shard_id ) self.__hash_sharding_specification_3 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_3.shard_id ) self.__hash_sharding_specification_4 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_4.shard_id ) self.__hash_sharding_specification_5 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_5.shard_id )
def setUp(self): self.manager, self.proxy = tests.utils.setup_xmlrpc() self.__options_1 = { "uuid": _uuid.UUID("{aa75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(0), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server1 = MySQLServer.discover_uuid(self.__options_1["address"]) self.__options_1["uuid"] = _uuid.UUID(uuid_server1) self.__server_1 = MySQLServer(**self.__options_1) MySQLServer.add(self.__server_1) self.__group_1 = Group("GROUPID1", "First description.") Group.add(self.__group_1) self.__group_1.add_server(self.__server_1) tests.utils.configure_decoupled_master(self.__group_1, self.__server_1) self.__options_2 = { "uuid": _uuid.UUID("{aa45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(1), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server2 = MySQLServer.discover_uuid(self.__options_2["address"]) self.__options_2["uuid"] = _uuid.UUID(uuid_server2) self.__server_2 = MySQLServer(**self.__options_2) MySQLServer.add(self.__server_2) self.__server_2.connect() self.__server_2.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_2.exec_stmt("CREATE DATABASE db1") self.__server_2.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_2.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_2 = Group("GROUPID2", "Second description.") Group.add(self.__group_2) self.__group_2.add_server(self.__server_2) tests.utils.configure_decoupled_master(self.__group_2, self.__server_2) self.__options_3 = { "uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(2), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"]) self.__options_3["uuid"] = _uuid.UUID(uuid_server3) self.__server_3 = MySQLServer(**self.__options_3) MySQLServer.add(self.__server_3) self.__server_3.connect() self.__server_3.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_3.exec_stmt("CREATE DATABASE db1") self.__server_3.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_3.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_3 = Group("GROUPID3", "Third description.") Group.add(self.__group_3) self.__group_3.add_server(self.__server_3) tests.utils.configure_decoupled_master(self.__group_3, self.__server_3) self.__options_4 = { "uuid": _uuid.UUID("{bb45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(3), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server4 = MySQLServer.discover_uuid(self.__options_4["address"]) self.__options_4["uuid"] = _uuid.UUID(uuid_server4) self.__server_4 = MySQLServer(**self.__options_4) MySQLServer.add(self.__server_4) self.__server_4.connect() self.__server_4.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_4.exec_stmt("CREATE DATABASE db1") self.__server_4.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_4.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_4 = Group("GROUPID4", "Fourth description.") Group.add(self.__group_4) self.__group_4.add_server(self.__server_4) tests.utils.configure_decoupled_master(self.__group_4, self.__server_4) self.__options_5 = { "uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(4), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"]) self.__options_5["uuid"] = _uuid.UUID(uuid_server5) self.__server_5 = MySQLServer(**self.__options_5) MySQLServer.add(self.__server_5) self.__server_5.connect() self.__server_5.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_5.exec_stmt("CREATE DATABASE db1") self.__server_5.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_5.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_5 = Group("GROUPID5", "Fifth description.") Group.add(self.__group_5) self.__group_5.add_server(self.__server_5) tests.utils.configure_decoupled_master(self.__group_5, self.__server_5) self.__options_6 = { "uuid": _uuid.UUID("{cc45b12b-98d1-414c-96af-9e9d4b179678}"), "address": MySQLInstances().get_address(5), "user": MySQLInstances().user, "passwd": MySQLInstances().passwd, } uuid_server6 = MySQLServer.discover_uuid(self.__options_6["address"]) self.__options_6["uuid"] = _uuid.UUID(uuid_server6) self.__server_6 = MySQLServer(**self.__options_6) MySQLServer.add(self.__server_6) self.__server_6.connect() self.__server_6.exec_stmt("DROP DATABASE IF EXISTS db1") self.__server_6.exec_stmt("CREATE DATABASE db1") self.__server_6.exec_stmt("CREATE TABLE db1.t1" "(userID INT, name VARCHAR(30))") for i in range(1, 501): self.__server_6.exec_stmt("INSERT INTO db1.t1 " "VALUES(%s, 'TEST %s')" % (i, i)) self.__group_6 = Group("GROUPID6", "Sixth description.") Group.add(self.__group_6) self.__group_6.add_server(self.__server_6) tests.utils.configure_decoupled_master(self.__group_6, self.__server_6) self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn() self.assertEquals(self.__shard_mapping_list, []) self.__shard_mapping_id_1 = ShardMapping.define("HASH", "GROUPID1") self.__shard_mapping_1 = ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID") self.__shard_1 = Shards.add("GROUPID2") self.__shard_2 = Shards.add("GROUPID3") self.__shard_3 = Shards.add("GROUPID4") self.__shard_4 = Shards.add("GROUPID5") self.__shard_5 = Shards.add("GROUPID6") self.__hash_sharding_specification_1 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_1.shard_id) self.__hash_sharding_specification_2 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_2.shard_id) self.__hash_sharding_specification_3 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_3.shard_id) self.__hash_sharding_specification_4 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_4.shard_id) self.__hash_sharding_specification_5 = HashShardingSpecification.add( self.__shard_mapping_1.shard_mapping_id, self.__shard_5.shard_id)