예제 #1
0
 def test_fetch_sharding_scheme(self):
     range_sharding_specifications = RangeShardingSpecification.list(1)
     self.assertTrue(ShardingUtils.compare_range_specifications(
         range_sharding_specifications[0],
         self.__range_sharding_specification_1))
     self.assertTrue(ShardingUtils.compare_range_specifications(
         range_sharding_specifications[1],
         self.__range_sharding_specification_2))
     self.assertTrue(ShardingUtils.compare_range_specifications(
         range_sharding_specifications[2],
         self.__range_sharding_specification_3))
예제 #2
0
 def test_fetch_sharding_scheme(self):
     range_sharding_specifications = RangeShardingSpecification.list(1)
     self.assertTrue(
         ShardingUtils.compare_range_specifications(
             range_sharding_specifications[0],
             self.__range_sharding_specification_1))
     self.assertTrue(
         ShardingUtils.compare_range_specifications(
             range_sharding_specifications[1],
             self.__range_sharding_specification_2))
     self.assertTrue(
         ShardingUtils.compare_range_specifications(
             range_sharding_specifications[2],
             self.__range_sharding_specification_3))
예제 #3
0
def _add_shard(shard_mapping_id, groupid_lb_list, state, update_only=False):
    """Add the RANGE shard specification. This represents a single instance
    of a shard specification that maps a key RANGE to a server.

    :param shard_mapping_id: The unique identification for a shard mapping.
    :param groupid_lb_list: The list of group_id, lower_bounds pairs in the
                        format, group_id/lower_bound, group_id/lower_bound... .
    :param state: Indicates whether a given shard is ENABLED or DISABLED
    :param update_only: Only update the state store and skip adding range checks.

    :return: True if the add succeeded.
                False otherwise.
    :raises: ShardingError If the group on which the shard is being
                           created does not exist,
                           If the shard_mapping_id is not found,
                           If adding the shard definition fails,
                           If the state of the shard is an invalid
                           value,
                           If the range definition is invalid.
    """
    shard_mapping = ShardMapping.fetch_shard_mapping_defn(shard_mapping_id)
    if shard_mapping is None:
        raise _errors.ShardingError(SHARD_MAPPING_NOT_FOUND % \
                                                    (shard_mapping_id,  ))

    schema_type = shard_mapping[1]

    if len(RangeShardingSpecification.list(shard_mapping_id)) != 0:
        raise _errors.ShardingError(SHARDS_ALREADY_EXIST)

    group_id_list, lower_bound_list = \
        _utils.get_group_lower_bound_list(groupid_lb_list)

    if (len(group_id_list) != len(lower_bound_list)) and\
        schema_type == "RANGE":
        raise _errors.ShardingError(LOWER_BOUND_GROUP_ID_COUNT_MISMATCH)

    if len(lower_bound_list) != 0 and schema_type == "HASH":
        raise _errors.ShardingError(LOWER_BOUND_AUTO_GENERATED)

    if schema_type in Shards.VALID_RANGE_SHARDING_TYPES:
        for lower_bound in lower_bound_list:
            if not SHARDING_DATATYPE_HANDLER[schema_type].\
                        is_valid_lower_bound(lower_bound):
                raise _errors.ShardingError(
                                INVALID_LOWER_BOUND_VALUE % (lower_bound, ))

    state = state.upper()
    if state not in Shards.VALID_SHARD_STATES:
        raise _errors.ShardingError(INVALID_SHARD_STATE % (state,  ))

    for index, group_id in enumerate(group_id_list):
        shard = Shards.add(group_id, state)

        shard_id = shard.shard_id

        if schema_type == "HASH":
            HashShardingSpecification.add(
                shard_mapping_id,
                shard_id
            )
            _LOGGER.debug(
                "Added Shard (map id = %s, id = %s).",
                shard_mapping_id,
                shard_id
            )
        else:
            range_sharding_specification = \
                SHARDING_SPECIFICATION_HANDLER[schema_type].add(
                                                shard_mapping_id,
                                                lower_bound_list[index],
                                                shard_id
                                            )
            _LOGGER.debug(
                "Added Shard (map id = %s, lower bound = %s, id = %s).",
                range_sharding_specification.shard_mapping_id,
                range_sharding_specification.lower_bound,
                range_sharding_specification.shard_id
            )

        if not update_only:
            #If the shard is added in a DISABLED state  do not setup replication
            #with the primary of the global group. Basically setup replication only
            #if the shard is ENABLED.
            if state == "ENABLED":
                _setup_shard_group_replication(shard_id)

    if not update_only:
        #Add the shard limits into the metadata present in each of the shards.
        _events.trigger_within_procedure(
            ADD_SHARD_RANGE_CHECK,
            shard_mapping_id,
            schema_type
        )
예제 #4
0
    def test_shard_split(self):
        status = self.proxy.sharding.split_shard("1", "GROUPID3", "600")
        self.assertStatus(status, _executor.Job.SUCCESS)
        self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
        self.assertEqual(status[1][-1]["description"],
                         "Executed action (_prune_shard_tables_after_split).")
        status = self.proxy.sharding.lookup_servers("db1.t1", 500,  "LOCAL")
        self.assertEqual(status[0], True)
        self.assertEqual(status[1], "")
        obtained_server_list = status[2]
        for idx in range(0, 2):
            server_uuid = obtained_server_list[idx][0]
            shard_server = MySQLServer.fetch(server_uuid)
            shard_server.connect()
            rows = shard_server.exec_stmt(
                                    "SELECT NAME FROM db1.t1",
                                    {"fetch" : True})
            self.assertEqual(len(rows), 3)
            self.assertEqual(rows[0][0], 'TEST 1')
            self.assertEqual(rows[1][0], 'TEST 2')
            self.assertEqual(rows[2][0], 'TEST 3')

        status = self.proxy.sharding.lookup_servers("db1.t1", 800,  "LOCAL")
        self.assertEqual(status[0], True)
        self.assertEqual(status[1], "")
        obtained_server_list = status[2]
        for idx in range(0, 2):
            server_uuid = obtained_server_list[idx][0]
            shard_server = MySQLServer.fetch(server_uuid)
            shard_server.connect()
            rows = shard_server.exec_stmt(
                                    "SELECT NAME FROM db1.t1",
                                    {"fetch" : True})
            self.assertEqual(len(rows), 4)
            self.assertEqual(rows[0][0], 'TEST 4')
            self.assertEqual(rows[1][0], 'TEST 5')
            self.assertEqual(rows[2][0], 'TEST 6')
            self.assertEqual(rows[3][0], 'TEST 7')

        status = self.proxy.sharding.lookup_servers("1", 500,  "GLOBAL")
        self.assertEqual(status[0], True)
        self.assertEqual(status[1], "")
        obtained_server_list = status[2]
        for idx in range(0, 2):
            if obtained_server_list[idx][2]:
                global_master_uuid = obtained_server_list[idx][0]
                break

        global_master = MySQLServer.fetch(global_master_uuid)
        global_master.connect()

        global_master.exec_stmt("DROP DATABASE IF EXISTS global_db")
        global_master.exec_stmt("CREATE DATABASE global_db")
        global_master.exec_stmt("CREATE TABLE global_db.global_table"
                                  "(userID INT, name VARCHAR(30))")
        global_master.exec_stmt("INSERT INTO global_db.global_table "
                                  "VALUES(101, 'TEST 1')")
        global_master.exec_stmt("INSERT INTO global_db.global_table "
                                  "VALUES(202, 'TEST 2')")

        status = self.proxy.group.promote("GROUPID1")
        self.assertStatus(status, _executor.Job.SUCCESS)
        self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
        self.assertEqual(status[1][-1]["description"],
                         "Executed action (_change_to_candidate).")

        sleep(5)

        status = self.proxy.sharding.lookup_servers("1", 500,  "GLOBAL")
        self.assertEqual(status[0], True)
        self.assertEqual(status[1], "")
        obtained_server_list = status[2]
        for idx in range(0, 2):
            if obtained_server_list[idx][2]:
                global_master_uuid = obtained_server_list[idx][0]
                break

        global_master = MySQLServer.fetch(global_master_uuid)
        global_master.connect()

        global_master.exec_stmt("INSERT INTO global_db.global_table "
                                  "VALUES(303, 'TEST 3')")
        global_master.exec_stmt("INSERT INTO global_db.global_table "
                                  "VALUES(404, 'TEST 4')")
        global_master.exec_stmt("INSERT INTO global_db.global_table "
                                  "VALUES(505, 'TEST 5')")
        global_master.exec_stmt("INSERT INTO global_db.global_table "
                                  "VALUES(606, 'TEST 6')")

        sleep(5)

        status = self.proxy.sharding.lookup_servers("db1.t1", 500,  "LOCAL")
        self.assertEqual(status[0], True)
        self.assertEqual(status[1], "")
        obtained_server_list = status[2]
        for idx in range(0, 2):
            if obtained_server_list[idx][2]:
                shard_uuid = obtained_server_list[idx][0]
                shard_server = MySQLServer.fetch(shard_uuid)
                shard_server.connect()
                rows = shard_server.exec_stmt(
                    "SELECT NAME FROM global_db.global_table", {"fetch" : True}
                )
                self.assertEqual(len(rows), 6)
                self.assertEqual(rows[0][0], 'TEST 1')
                self.assertEqual(rows[1][0], 'TEST 2')
                self.assertEqual(rows[2][0], 'TEST 3')
                self.assertEqual(rows[3][0], 'TEST 4')
                self.assertEqual(rows[4][0], 'TEST 5')
                self.assertEqual(rows[5][0], 'TEST 6')

        #Ensure tha two new shard_ids have been generated.
        range_sharding_specifications = RangeShardingSpecification.list(1)
        self.assertTrue(ShardingUtils.compare_range_specifications(
            range_sharding_specifications[0],
            RangeShardingSpecification.fetch(2)))
        self.assertTrue(ShardingUtils.compare_range_specifications(
            range_sharding_specifications[1],
            RangeShardingSpecification.fetch(3)))
예제 #5
0
    def test_shard_split(self):
        status = self.proxy.sharding.split_shard("1", "GROUPID3", "600")
        self.check_xmlrpc_command_result(status)
        status = self.proxy.sharding.lookup_servers("db1.t1", 500,  "LOCAL")
        for info in self.check_xmlrpc_iter(status):
            server_uuid = info['server_uuid']
            shard_server = fetch_test_server(server_uuid)
            shard_server.connect()
            rows = shard_server.exec_stmt(
                                    "SELECT NAME FROM db1.t1",
                                    {"fetch" : True})
            self.assertEqual(len(rows), 3)
            self.assertEqual(rows[0][0], 'TEST 1')
            self.assertEqual(rows[1][0], 'TEST 2')
            self.assertEqual(rows[2][0], 'TEST 3')

        status = self.proxy.sharding.lookup_servers("db1.t1", 800,  "LOCAL")
        for info in self.check_xmlrpc_iter(status):
            server_uuid = info['server_uuid']
            shard_server = fetch_test_server(server_uuid)
            shard_server.connect()
            rows = shard_server.exec_stmt(
                                    "SELECT NAME FROM db1.t1",
                                    {"fetch" : True})
            self.assertEqual(len(rows), 4)
            self.assertEqual(rows[0][0], 'TEST 4')
            self.assertEqual(rows[1][0], 'TEST 5')
            self.assertEqual(rows[2][0], 'TEST 6')
            self.assertEqual(rows[3][0], 'TEST 7')

        status = self.proxy.sharding.lookup_servers("1", 500,  "GLOBAL")
        for info in self.check_xmlrpc_iter(status):
            if info['status'] == MySQLServer.PRIMARY:
                global_master = fetch_test_server(info['server_uuid'])
                global_master.connect()

        global_master.exec_stmt("DROP DATABASE IF EXISTS global_db")
        global_master.exec_stmt("CREATE DATABASE global_db")
        global_master.exec_stmt("CREATE TABLE global_db.global_table"
                                  "(userID INT, name VARCHAR(30))")
        global_master.exec_stmt("INSERT INTO global_db.global_table "
                                  "VALUES(101, 'TEST 1')")
        global_master.exec_stmt("INSERT INTO global_db.global_table "
                                  "VALUES(202, 'TEST 2')")

        status = self.proxy.group.promote("GROUPID1")
        self.check_xmlrpc_command_result(status)

        sleep(5)

        status = self.proxy.sharding.lookup_servers("1", 500,  "GLOBAL")
        for info in self.check_xmlrpc_iter(status):
            if info['status'] == MySQLServer.PRIMARY:
                global_master = fetch_test_server(info['server_uuid'])
                global_master.connect()

        global_master.exec_stmt("INSERT INTO global_db.global_table "
                                  "VALUES(303, 'TEST 3')")
        global_master.exec_stmt("INSERT INTO global_db.global_table "
                                  "VALUES(404, 'TEST 4')")
        global_master.exec_stmt("INSERT INTO global_db.global_table "
                                  "VALUES(505, 'TEST 5')")
        global_master.exec_stmt("INSERT INTO global_db.global_table "
                                  "VALUES(606, 'TEST 6')")

        sleep(5)

        status = self.proxy.sharding.lookup_servers("db1.t1", 500,  "LOCAL")
        for info in self.check_xmlrpc_iter(status):
            if info['status'] == MySQLServer.PRIMARY:
                shard_server = fetch_test_server(info['server_uuid'])
                shard_server.connect()
                rows = shard_server.exec_stmt(
                    "SELECT NAME FROM global_db.global_table", {"fetch" : True}
                )
                self.assertEqual(len(rows), 6)
                self.assertEqual(rows[0][0], 'TEST 1')
                self.assertEqual(rows[1][0], 'TEST 2')
                self.assertEqual(rows[2][0], 'TEST 3')
                self.assertEqual(rows[3][0], 'TEST 4')
                self.assertEqual(rows[4][0], 'TEST 5')
                self.assertEqual(rows[5][0], 'TEST 6')

        #Ensure tha two new shard_ids have been generated.
        range_sharding_specifications = RangeShardingSpecification.list(1)
        self.assertTrue(ShardingUtils.compare_range_specifications(
            range_sharding_specifications[0],
            RangeShardingSpecification.fetch(2)))
        self.assertTrue(ShardingUtils.compare_range_specifications(
            range_sharding_specifications[1],
            RangeShardingSpecification.fetch(3)))
예제 #6
0
def _add_shard(shard_mapping_id, groupid_lb_list, state):
    """Add the RANGE shard specification. This represents a single instance
    of a shard specification that maps a key RANGE to a server.

    :param shard_mapping_id: The unique identification for a shard mapping.
    :param groupid_lb_list: The list of group_id, lower_bounds pairs in the
                        format, group_id/lower_bound, group_id/lower_bound... .
    :param state: Indicates whether a given shard is ENABLED or DISABLED

    :return: True if the add succeeded.
                False otherwise.
    :raises: ShardingError If the group on which the shard is being
                           created does not exist,
                           If the shard_mapping_id is not found,
                           If adding the shard definition fails,
                           If the state of the shard is an invalid
                           value,
                           If the range definition is invalid.
    """
    shard_mapping = ShardMapping.fetch_shard_mapping_defn(shard_mapping_id)
    if shard_mapping is None:
        raise _errors.ShardingError(SHARD_MAPPING_NOT_FOUND % \
                                                    (shard_mapping_id,  ))

    schema_type = shard_mapping[1]

    if len(RangeShardingSpecification.list(shard_mapping_id)) != 0:
        raise _errors.ShardingError(SHARDS_ALREADY_EXIST)

    group_id_list, lower_bound_list = \
        _utils.get_group_lower_bound_list(groupid_lb_list)

    if (len(group_id_list) != len(lower_bound_list)) and\
        schema_type == "RANGE":
        raise _errors.ShardingError(LOWER_BOUND_GROUP_ID_COUNT_MISMATCH)

    if len(lower_bound_list) != 0 and schema_type == "HASH":
        raise _errors.ShardingError(LOWER_BOUND_AUTO_GENERATED)

    if schema_type in Shards.VALID_RANGE_SHARDING_TYPES:
        for lower_bound in lower_bound_list:
            if not SHARDING_DATATYPE_HANDLER[schema_type].\
                        is_valid_lower_bound(lower_bound):
                raise _errors.ShardingError(INVALID_LOWER_BOUND_VALUE %
                                            (lower_bound, ))

    state = state.upper()
    if state not in Shards.VALID_SHARD_STATES:
        raise _errors.ShardingError(INVALID_SHARD_STATE % (state, ))

    for index, group_id in enumerate(group_id_list):
        shard = Shards.add(group_id, state)

        shard_id = shard.shard_id

        if schema_type == "HASH":
            HashShardingSpecification.add(shard_mapping_id, shard_id)
            _LOGGER.debug("Added Shard (map id = %s, id = %s).",
                          shard_mapping_id, shard_id)
        else:
            range_sharding_specification = \
                SHARDING_SPECIFICATION_HANDLER[schema_type].add(
                                                shard_mapping_id,
                                                lower_bound_list[index],
                                                shard_id
                                            )
            _LOGGER.debug(
                "Added Shard (map id = %s, lower bound = %s, id = %s).",
                range_sharding_specification.shard_mapping_id,
                range_sharding_specification.lower_bound,
                range_sharding_specification.shard_id)

        #If the shard is added in a DISABLED state  do not setup replication
        #with the primary of the global group. Basically setup replication only
        #if the shard is ENABLED.
        if state == "ENABLED":
            _setup_shard_group_replication(shard_id)