Ejemplo n.º 1
0
    def setUp(self):
        """Configure the existing environment
        """
        self.manager, self.proxy = tests.utils.setup_xmlrpc()
        self.__options_1 = {
            "uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
            "address": "server_1.mysql.com:3060",
            "user": MySQLInstances().user,
            "passwd": MySQLInstances().passwd,
        }
        self.__server_1 = MySQLServer(**self.__options_1)
        MySQLServer.add(self.__server_1)
        self.__options_2 = {
            "uuid": _uuid.UUID("{aa75a12a-98d1-414c-96af-9e9d4b179678}"),
            "address": "server_2.mysql.com:3060",
            "user": MySQLInstances().user,
            "passwd": MySQLInstances().passwd,
        }
        self.__server_2 = MySQLServer(**self.__options_2)
        MySQLServer.add(self.__server_2)
        self.__group_1 = Group("GROUPID1", "First description.")
        Group.add(self.__group_1)
        self.__group_1.add_server(self.__server_1)
        self.__group_1.add_server(self.__server_2)

        self.__options_3 = {
            "uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
            "address": MySQLInstances().get_address(0),
            "user": MySQLInstances().user,
            "passwd": MySQLInstances().passwd,
        }
        uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
        self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
        self.__server_3 = MySQLServer(**self.__options_3)
        MySQLServer.add(self.__server_3)

        self.__options_4 = {
            "uuid": _uuid.UUID("{dd75a12a-98d1-414c-96af-9e9d4b179678}"),
            "address": "server_4.mysql.com:3060",
            "user": MySQLInstances().user,
            "passwd": MySQLInstances().passwd,
        }
        self.__server_4 = MySQLServer(**self.__options_4)
        MySQLServer.add(self.__server_4)
        self.__group_2 = Group("GROUPID2", "Second description.")
        Group.add(self.__group_2)
        self.__group_2.add_server(self.__server_3)
        self.__group_2.add_server(self.__server_4)
        tests.utils.configure_decoupled_master(self.__group_2, self.__server_3)

        self.__options_5 = {
            "uuid": _uuid.UUID("{ee75b12b-98d1-414c-96af-9e9d4b179678}"),
            "address": MySQLInstances().get_address(2),
            "user": MySQLInstances().user,
            "passwd": MySQLInstances().passwd,
        }
        uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
        self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
        self.__server_5 = MySQLServer(**self.__options_5)
        MySQLServer.add(self.__server_5)

        self.__options_6 = {
            "uuid": _uuid.UUID("{ff75a12a-98d1-414c-96af-9e9d4b179678}"),
            "address": "server_6.mysql.com:3060",
            "user": MySQLInstances().user,
            "passwd": MySQLInstances().passwd,
        }
        self.__server_6 = MySQLServer(**self.__options_6)
        MySQLServer.add(self.__server_6)
        self.__group_3 = Group("GROUPID3", "Third description.")
        Group.add(self.__group_3)
        self.__group_3.add_server(self.__server_5)
        self.__group_3.add_server(self.__server_6)
        tests.utils.configure_decoupled_master(self.__group_3, self.__server_5)

        self.__options_1_host,  self.__options_1_port = \
            server_utils.split_host_port(self.__options_1["address"], 13001)
        self.__options_2_host,  self.__options_2_port = \
            server_utils.split_host_port(self.__options_2["address"], 13001)
        self.__options_3_host,  self.__options_3_port = \
            server_utils.split_host_port(self.__options_3["address"], 13001)
        self.__options_4_host,  self.__options_4_port = \
            server_utils.split_host_port(self.__options_4["address"], 13001)
        self.__options_5_host,  self.__options_5_port = \
            server_utils.split_host_port(self.__options_5["address"], 13001)
        self.__options_6_host,  self.__options_6_port = \
            server_utils.split_host_port(self.__options_6["address"], 13001)

        group_4 = Group("GROUPID4", "4TH description.")
        Group.add(group_4)
        group_5 = Group("GROUPID5", "5TH description.")
        Group.add(group_5)
        group_6 = Group("GROUPID6", "6TH description.")
        Group.add(group_6)
        group_7 = Group("GROUPID7", "7TH description.")
        Group.add(group_7)
        group_8 = Group("GROUPID8", "8TH description.")
        Group.add(group_8)
        group_9 = Group("GROUPID9", "9TH description.")
        Group.add(group_9)
        group_10 = Group("GROUPID10", "10TH description.")
        Group.add(group_10)
        group_11 = Group("GROUPID11", "11TH description.")
        Group.add(group_11)
        group_12 = Group("GROUPID12", "12TH description.")
        Group.add(group_12)
        group_13 = Group("GROUPID13", "13TH description.")
        Group.add(group_13)
        group_14 = Group("GROUPID14", "14TH description.")
        Group.add(group_14)

        self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
        self.assertEquals(self.__shard_mapping_list, [])
        self.__shard_mapping_id_1 = ShardMapping.define("RANGE", "GROUPID10")
        self.__shard_mapping_id_2 = ShardMapping.define("RANGE", "GROUPID11")
        self.__shard_mapping_id_3 = ShardMapping.define("RANGE", "GROUPID12")
        #Test with sharding type values in lower case
        self.__shard_mapping_id_4 = ShardMapping.define("range", "GROUPID13")
        self.__shard_mapping_id_5 = ShardMapping.define("range", "GROUPID14")

        self.__shard_mapping_1 = \
            ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID1")
        self.__shard_mapping_2 = \
            ShardMapping.add(self.__shard_mapping_id_2, "db2.t2", "userID2")
        self.__shard_mapping_3 = \
            ShardMapping.add(self.__shard_mapping_id_3, "db3.t3", "userID3")
        self.__shard_mapping_4 = \
            ShardMapping.add(self.__shard_mapping_id_4, "db4.t4", "userID4")

        self.__shard_mapping_5 = \
            ShardMapping.add(self.__shard_mapping_id_5, "prune_db.prune_table",
                             "userID")

        self.__shard_id_1 = Shards.add("GROUPID1", "ENABLED")
        self.__shard_id_2 = Shards.add("GROUPID10", "ENABLED")
        self.__shard_id_3 = Shards.add("GROUPID11", "DISABLED")
        self.__shard_id_4 = Shards.add("GROUPID4", "ENABLED")
        self.__shard_id_5 = Shards.add("GROUPID5", "ENABLED")
        self.__shard_id_6 = Shards.add("GROUPID6", "ENABLED")
        self.__shard_id_7 = Shards.add("GROUPID7", "ENABLED")
        self.__shard_id_8 = Shards.add("GROUPID8", "ENABLED")
        self.__shard_id_9 = Shards.add("GROUPID9", "ENABLED")
        self.__shard_id_10 = Shards.add("GROUPID2", "ENABLED")
        self.__shard_id_11 = Shards.add("GROUPID3", "ENABLED")

        self.__range_sharding_specification_1 = RangeShardingSpecification.add(
            self.__shard_mapping_1.shard_mapping_id, 0,
            self.__shard_id_1.shard_id)
        self.__range_sharding_specification_2 = RangeShardingSpecification.add(
            self.__shard_mapping_1.shard_mapping_id, 1001,
            self.__shard_id_2.shard_id)
        self.__range_sharding_specification_3 = RangeShardingSpecification.add(
            self.__shard_mapping_1.shard_mapping_id, 2001,
            self.__shard_id_3.shard_id)

        self.__range_sharding_specification_4 = RangeShardingSpecification.add(
            self.__shard_mapping_2.shard_mapping_id, 3001,
            self.__shard_id_4.shard_id)
        self.__range_sharding_specification_5 = RangeShardingSpecification.add(
            self.__shard_mapping_2.shard_mapping_id, 4001,
            self.__shard_id_5.shard_id)

        self.__range_sharding_specification_6 = RangeShardingSpecification.add(
            self.__shard_mapping_3.shard_mapping_id, 6001,
            self.__shard_id_6.shard_id)
        self.__range_sharding_specification_7 = RangeShardingSpecification.add(
            self.__shard_mapping_3.shard_mapping_id, 7001,
            self.__shard_id_7.shard_id)

        self.__range_sharding_specification_8 = RangeShardingSpecification.add(
            self.__shard_mapping_4.shard_mapping_id, 8001,
            self.__shard_id_8.shard_id)
        self.__range_sharding_specification_9 = RangeShardingSpecification.add(
            self.__shard_mapping_4.shard_mapping_id, 10001,
            self.__shard_id_9.shard_id)

        self.__range_sharding_specification_10 = RangeShardingSpecification.add(
            self.__shard_mapping_5.shard_mapping_id, 100,
            self.__shard_id_10.shard_id)
        self.__range_sharding_specification_11 = RangeShardingSpecification.add(
            self.__shard_mapping_5.shard_mapping_id, 201,
            self.__shard_id_11.shard_id)

        READ_ONLY = MySQLServer.get_mode_idx(MySQLServer.READ_ONLY)
        READ_WRITE = MySQLServer.get_mode_idx(MySQLServer.READ_WRITE)

        SECONDARY = MySQLServer.get_status_idx(MySQLServer.SECONDARY)
        PRIMARY = MySQLServer.get_status_idx(MySQLServer.PRIMARY)

        self.__setofservers = tests.utils.make_servers_result(
            [[
                str(self.__server_1.uuid), 'GROUPID1', self.__options_1_host,
                self.__options_1_port, READ_ONLY, SECONDARY, 1.0
            ],
             [
                 str(self.__server_2.uuid), 'GROUPID1', self.__options_2_host,
                 self.__options_2_port, READ_ONLY, SECONDARY, 1.0
             ],
             [
                 str(self.__server_3.uuid), 'GROUPID2', self.__options_3_host,
                 self.__options_3_port, READ_WRITE, PRIMARY, 1.0
             ],
             [
                 str(self.__server_4.uuid), 'GROUPID2', self.__options_4_host,
                 self.__options_4_port, READ_ONLY, SECONDARY, 1.0
             ],
             [
                 str(self.__server_5.uuid), 'GROUPID3', self.__options_5_host,
                 self.__options_5_port, READ_WRITE, PRIMARY, 1.0
             ],
             [
                 str(self.__server_6.uuid), 'GROUPID3', self.__options_6_host,
                 self.__options_6_port, READ_ONLY, SECONDARY, 1.0
             ]])

        self.__setofservers_1 = tests.utils.make_servers_result(
            [[
                str(self.__server_1.uuid), 'GROUPID1', self.__options_1_host,
                self.__options_1_port, READ_ONLY, SECONDARY, 1.0
            ],
             [
                 str(self.__server_2.uuid), 'GROUPID1', self.__options_2_host,
                 self.__options_2_port, READ_ONLY, SECONDARY, 1.0
             ]])

        self.__setofservers_2 = tests.utils.make_servers_result(
            [[
                str(self.__server_1.uuid), 'GROUPID1', self.__options_1_host,
                self.__options_1_port, READ_ONLY, SECONDARY, 1.0
            ],
             [
                 str(self.__server_2.uuid), 'GROUPID1', self.__options_2_host,
                 self.__options_2_port, READ_ONLY, SECONDARY, 1.0
             ],
             [
                 str(self.__server_3.uuid), 'GROUPID2', self.__options_3_host,
                 self.__options_3_port, READ_WRITE, PRIMARY, 1.0
             ],
             [
                 str(self.__server_4.uuid), 'GROUPID2', self.__options_4_host,
                 self.__options_4_port, READ_ONLY, SECONDARY, 1.0
             ]])

        self.__setofservers_3 = tests.utils.make_servers_result(
            [[
                str(self.__server_1.uuid), 'GROUPID1', self.__options_1_host,
                self.__options_1_port, READ_ONLY, SECONDARY, 1.0
            ],
             [
                 str(self.__server_2.uuid), 'GROUPID1', self.__options_2_host,
                 self.__options_2_port, READ_ONLY, SECONDARY, 1.0
             ],
             [
                 str(self.__server_3.uuid), 'GROUPID2', self.__options_3_host,
                 self.__options_3_port, READ_WRITE, PRIMARY, 1.0
             ],
             [
                 str(self.__server_4.uuid), 'GROUPID2', self.__options_4_host,
                 self.__options_4_port, READ_ONLY, SECONDARY, 1.0
             ],
             [
                 str(self.__server_5.uuid), 'GROUPID3', self.__options_5_host,
                 self.__options_5_port, READ_WRITE, PRIMARY, 1.0
             ],
             [
                 str(self.__server_6.uuid), 'GROUPID3', self.__options_6_host,
                 self.__options_6_port, READ_ONLY, SECONDARY, 1.0
             ]])

        self.__setoftables = tests.utils.make_tables_result(
            [['db1', 't1', 'userID1', '1'], ['db2', 't2', 'userID2', '2'],
             ['db3', 't3', 'userID3', '3'], ['db4', 't4', 'userID4', '4'],
             ['prune_db', 'prune_table', 'userID', '5']])
        self.__setoftables_1 = tests.utils.make_tables_result(
            [['db1', 't1', 'userID1', '1']])
        self.__setoftables_2 = tests.utils.make_tables_result(
            [['db1', 't1', 'userID1', '1'], ['db2', 't2', 'userID2', '2']])
        self.__setoftables_3 = tests.utils.make_tables_result(
            [['db1', 't1', 'userID1', '1'], ['db2', 't2', 'userID2', '2'],
             ['db3', 't3', 'userID3', '3']])
        self.__setofshardmaps = tests.utils.make_mapping_result(
            [['1', 'RANGE', 'GROUPID10'], ['2', 'RANGE', 'GROUPID11'],
             ['3', 'RANGE', 'GROUPID12'], ['4', 'RANGE', 'GROUPID13'],
             ['5', 'RANGE', 'GROUPID14']])
        self.__setofshardmaps_1 = tests.utils.make_mapping_result(
            [['1', 'RANGE', 'GROUPID10']])
        self.__setofshardmaps_2 = tests.utils.make_mapping_result(
            [['1', 'RANGE', 'GROUPID10'], ['2', 'RANGE', 'GROUPID11']])
        self.__setofshardmaps_3 = tests.utils.make_mapping_result(
            [['1', 'RANGE', 'GROUPID10'], ['2', 'RANGE', 'GROUPID11'],
             ['3', 'RANGE', 'GROUPID12']])
        self.__setofshardindexes = tests.utils.make_index_result(
            [['0', '1', '1', 'GROUPID1'], ['1001', '1', '2', 'GROUPID10'],
             ['3001', '2', '4', 'GROUPID4'], ['4001', '2', '5', 'GROUPID5'],
             ['6001', '3', '6', 'GROUPID6'], ['7001', '3', '7', 'GROUPID7'],
             ['8001', '4', '8', 'GROUPID8'], ['10001', '4', '9', 'GROUPID9'],
             ['100', '5', '10', 'GROUPID2'], ['201', '5', '11', 'GROUPID3']])
        self.__setofshardindexes_1 = tests.utils.make_index_result(
            [['0', '1', '1', 'GROUPID1'], ['1001', '1', '2', 'GROUPID10']])
        self.__setofshardindexes_3 = tests.utils.make_index_result(
            [['0', '1', '1', 'GROUPID1'], ['1001', '1', '2', 'GROUPID10'],
             ['3001', '2', '4', 'GROUPID4'], ['4001', '2', '5', 'GROUPID5'],
             ['6001', '3', '6', 'GROUPID6'], ['7001', '3', '7', 'GROUPID7']])
        self.__setofshardindexes_5 = tests.utils.make_index_result(
            [['0', '1', '1', 'GROUPID1'], ['1001', '1', '2', 'GROUPID10'],
             ['3001', '2', '4', 'GROUPID4'], ['4001', '2', '5', 'GROUPID5'],
             ['6001', '3', '6', 'GROUPID6'], ['7001', '3', '7', 'GROUPID7'],
             ['8001', '4', '8', 'GROUPID8'], ['10001', '4', '9', 'GROUPID9'],
             ['100', '5', '10', 'GROUPID2'], ['201', '5', '11', 'GROUPID3']])
        self.__shardinginformation_1 = tests.utils.make_info_result(
            [[
                'db1', 't1', 'userID1', '0', '1', 'RANGE', 'GROUPID1',
                'GROUPID10'
            ],
             [
                 'db1', 't1', 'userID1', '1001', '2', 'RANGE', 'GROUPID10',
                 'GROUPID10'
             ]])
        self.__shardinginformation_2 = tests.utils.make_info_result(
            [[
                'db1', 't1', 'userID1', '0', '1', 'RANGE', 'GROUPID1',
                'GROUPID10'
            ],
             [
                 'db1', 't1', 'userID1', '1001', '2', 'RANGE', 'GROUPID10',
                 'GROUPID10'
             ],
             [
                 'db2', 't2', 'userID2', '3001', '4', 'RANGE', 'GROUPID4',
                 'GROUPID11'
             ],
             [
                 'db2', 't2', 'userID2', '4001', '5', 'RANGE', 'GROUPID5',
                 'GROUPID11'
             ]])
        self.__shardinginformation_3 = tests.utils.make_info_result(
            [[
                'db1', 't1', 'userID1', '0', '1', 'RANGE', 'GROUPID1',
                'GROUPID10'
            ],
             [
                 'db1', 't1', 'userID1', '1001', '2', 'RANGE', 'GROUPID10',
                 'GROUPID10'
             ],
             [
                 'db2', 't2', 'userID2', '3001', '4', 'RANGE', 'GROUPID4',
                 'GROUPID11'
             ],
             [
                 'db2', 't2', 'userID2', '4001', '5', 'RANGE', 'GROUPID5',
                 'GROUPID11'
             ],
             [
                 'db3', 't3', 'userID3', '6001', '6', 'RANGE', 'GROUPID6',
                 'GROUPID12'
             ],
             [
                 'db3', 't3', 'userID3', '7001', '7', 'RANGE', 'GROUPID7',
                 'GROUPID12'
             ]])
Ejemplo n.º 2
0
def _setup_shard_switch_split(shard_id, source_group_id, destination_group_id,
                              split_value, prune_limit, cmd, update_only):
    """Setup the moved shard to map to the new group.

    :param shard_id: The shard ID of the shard that needs to be moved.
    :param source_group_id: The group_id of the source shard.
    :param destn_group_id: The ID of the group to which the shard needs to
                           be moved.
    :param split_value: Indicates the value at which the range for the
                        particular shard will be split. Will be set only
                        for shard split operations.
    :param prune_limit: The number of DELETEs that should be
                        done in one batch.
    :param cmd: Indicates the type of re-sharding operation.
    :update_only: Only update the state store and skip provisioning.
    """
    #Fetch the Range sharding specification.
    range_sharding_spec, source_shard, shard_mappings, shard_mapping_defn = \
            _services_sharding.verify_and_fetch_shard(shard_id)

    #Disable the old shard
    source_shard.disable()

    #Remove the old shard.
    range_sharding_spec.remove()
    source_shard.remove()

    destination_group = Group.fetch(destination_group_id)
    if destination_group is None:
        raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
                                    (destination_group_id, ))
    destn_group_master = MySQLServer.fetch(destination_group.master)
    if destn_group_master is None:
        raise _errors.ShardingError(
            _services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
    destn_group_master.connect()

    #Make the destination group as read only to disable updates until the
    #connectors update their caches, thus avoiding inconsistency.
    destn_group_master.read_only = True

    #Add the new shards. Generate new shard IDs for the shard being
    #split and also for the shard that is created as a result of the split.
    new_shard_1 = Shards.add(source_shard.group_id, "DISABLED")
    new_shard_2 = Shards.add(destination_group_id, "DISABLED")

    #Both of the shard mappings associated with this shard_id should
    #be of the same sharding type. Hence it is safe to use one of the
    #shard mappings.
    if shard_mappings[0].type_name == "HASH":
        #In the case of a split involving a HASH sharding scheme,
        #the shard that is split gets a new shard_id, while the split
        #gets the new computed lower_bound and also a new shard id.
        #NOTE: How the shard that is split retains its lower_bound.
        HashShardingSpecification.add_hash_split(
            range_sharding_spec.shard_mapping_id,
            new_shard_1.shard_id,
            range_sharding_spec.lower_bound
        )
        HashShardingSpecification.add_hash_split(
            range_sharding_spec.shard_mapping_id,
            new_shard_2.shard_id,
            split_value
        )
    else:
        #Add the new ranges. Note that the shard being split retains
        #its lower_bound, while the new shard gets the computed,
        #lower_bound.
        RangeShardingSpecification.add(
            range_sharding_spec.shard_mapping_id,
            range_sharding_spec.lower_bound,
            new_shard_1.shard_id
        )
        RangeShardingSpecification.add(
            range_sharding_spec.shard_mapping_id,
            split_value,
            new_shard_2.shard_id
        )

    #The sleep ensures that the connector have refreshed their caches with the
    #new shards that have been added as a result of the split.
    time.sleep(_utils.TTL)

    #The source shard group master would have been marked as read only
    #during the sync. Remove the read_only flag.
    source_group = Group.fetch(source_group_id)
    if source_group is None:
        raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
                                    (source_group_id, ))

    source_group_master = MySQLServer.fetch(source_group.master)
    if source_group_master is None:
        raise _errors.ShardingError(
            _services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
    source_group_master.connect()

    #Kill all the existing connections on the servers
    source_group.kill_connections_on_servers()

    #Allow connections on the source group master
    source_group_master.read_only = False

    #Allow connections on the destination group master
    destn_group_master.read_only = False

    #Setup replication for the new group from the global server
    _group_replication.setup_group_replication \
            (shard_mapping_defn[2], destination_group_id)

    #Enable the split shards
    new_shard_1.enable()
    new_shard_2.enable()

    #Trigger changing the mappings for the shard that was copied
    if not update_only:
        _events.trigger_within_procedure(
            PRUNE_SHARDS, new_shard_1.shard_id, new_shard_2.shard_id, prune_limit
        )
Ejemplo n.º 3
0
    def setUp(self):
        """Configure the existing environment
        """
        self.__options_1 = {
            "uuid" :  _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
            "address"  : "server_1.mysql.com:3060",
        }
        self.__server_1 = MySQLServer(**self.__options_1)
        MySQLServer.add(self.__server_1)
        self.__options_2 = {
            "uuid" :  _uuid.UUID("{aa75a12a-98d1-414c-96af-9e9d4b179678}"),
            "address"  : "server_2.mysql.com:3060",
        }
        self.__server_2 = MySQLServer(**self.__options_2)
        MySQLServer.add(self.__server_2)
        self.__group_1 = Group("GROUPID1", "First description.")
        Group.add(self.__group_1)
        self.__group_1.add_server(self.__server_1)
        self.__group_1.add_server(self.__server_2)

        self.__options_3 = {
            "uuid" :  _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
            "address"  : MySQLInstances().get_address(0),
            "user" : MySQLInstances().user,
            "passwd" : MySQLInstances().passwd,
        }
        uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
        self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
        self.__server_3 = MySQLServer(**self.__options_3)
        MySQLServer.add(self.__server_3)
        self.__server_3.connect()
        self.__server_3.exec_stmt("DROP DATABASE IF EXISTS prune_db")
        self.__server_3.exec_stmt("CREATE DATABASE prune_db")
        self.__server_3.exec_stmt("CREATE TABLE prune_db.prune_table"
                                  "(userID INT, name VARCHAR(30))")
        self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table "
                                  "VALUES(101, 'TEST 1')")
        self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table "
                                  "VALUES(202, 'TEST 2')")

        self.__options_4 = {
            "uuid" :  _uuid.UUID("{dd75a12a-98d1-414c-96af-9e9d4b179678}"),
            "address"  : "server_4.mysql.com:3060",
        }
        self.__server_4 = MySQLServer(**self.__options_4)
        MySQLServer.add(self.__server_4)
        self.__group_2 = Group("GROUPID2", "Second description.")
        Group.add(self.__group_2)
        self.__group_2.add_server(self.__server_3)
        self.__group_2.add_server(self.__server_4)
        tests.utils.configure_decoupled_master(self.__group_2, self.__server_3)

        self.__options_5 = {
            "uuid" :  _uuid.UUID("{ee75b12b-98d1-414c-96af-9e9d4b179678}"),
            "address"  : MySQLInstances().get_address(2),
            "user" : MySQLInstances().user,
            "passwd" : MySQLInstances().passwd,
        }
        uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
        self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
        self.__server_5 = MySQLServer(**self.__options_5)
        MySQLServer.add(self.__server_5)
        self.__server_5.connect()
        self.__server_5.exec_stmt("DROP DATABASE IF EXISTS prune_db")
        self.__server_5.exec_stmt("CREATE DATABASE prune_db")
        self.__server_5.exec_stmt("CREATE TABLE prune_db.prune_table"
                                  "(userID INT, name VARCHAR(30))")
        self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table "
                                  "VALUES(101, 'TEST 1')")
        self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table "
                                  "VALUES(202, 'TEST 2')")

        self.__options_6 = {
            "uuid" :  _uuid.UUID("{ff75a12a-98d1-414c-96af-9e9d4b179678}"),
            "address"  : "server_6.mysql.com:3060",
        }
        self.__server_6 = MySQLServer(**self.__options_6)
        MySQLServer.add(self.__server_6)
        self.__group_3 = Group("GROUPID3", "Third description.")
        Group.add(self.__group_3)
        self.__group_3.add_server(self.__server_5)
        self.__group_3.add_server(self.__server_6)
        tests.utils.configure_decoupled_master(self.__group_3, self.__server_5)

        group_4 = Group("GROUPID4", "4TH description.")
        Group.add(group_4)
        group_5 = Group("GROUPID5", "5TH description.")
        Group.add(group_5)
        group_6 = Group("GROUPID6", "6TH description.")
        Group.add(group_6)
        group_7 = Group("GROUPID7", "7TH description.")
        Group.add(group_7)
        group_8 = Group("GROUPID8", "8TH description.")
        Group.add(group_8)
        group_9 = Group("GROUPID9", "9TH description.")
        Group.add(group_9)
        group_10 = Group("GROUPID10", "10TH description.")
        Group.add(group_10)
        group_11 = Group("GROUPID11", "11TH description.")
        Group.add(group_11)
        group_12 = Group("GROUPID12", "12TH description.")
        Group.add(group_12)
        group_13 = Group("GROUPID13", "13TH description.")
        Group.add(group_13)
        group_14 = Group("GROUPID14", "14TH description.")
        Group.add(group_14)

        self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
        self.assertEquals( self.__shard_mapping_list,  [])
        self.__shard_mapping_id_1 = ShardMapping.define("RANGE", "GROUPID10")
        self.__shard_mapping_id_2 = ShardMapping.define("RANGE", "GROUPID11")
        self.__shard_mapping_id_3 = ShardMapping.define("RANGE", "GROUPID12")
        #Test with sharding type values in lower case
        self.__shard_mapping_id_4 = ShardMapping.define("range", "GROUPID13")
        self.__shard_mapping_id_5 = ShardMapping.define("range", "GROUPID14")

        self.__shard_mapping_1 = \
            ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID1")
        self.__shard_mapping_2 = \
            ShardMapping.add(self.__shard_mapping_id_2, "db2.t2", "userID2")
        self.__shard_mapping_3 = \
            ShardMapping.add(self.__shard_mapping_id_3, "db3.t3", "userID3")
        self.__shard_mapping_4 = \
            ShardMapping.add(self.__shard_mapping_id_4, "db4.t4", "userID4")

        self.__shard_mapping_5 = \
            ShardMapping.add(self.__shard_mapping_id_5, "prune_db.prune_table",
                             "userID")

        self.__shard_id_1 = Shards.add("GROUPID1")
        self.__shard_id_2 = Shards.add("GROUPID10")
        self.__shard_id_3 = Shards.add("GROUPID11")
        self.__shard_id_4 = Shards.add("GROUPID4")
        self.__shard_id_5 = Shards.add("GROUPID5")
        self.__shard_id_6 = Shards.add("GROUPID6")
        self.__shard_id_7 = Shards.add("GROUPID7")
        self.__shard_id_8 = Shards.add("GROUPID8")
        self.__shard_id_9 = Shards.add("GROUPID9")
        self.__shard_id_10 = Shards.add("GROUPID2")
        self.__shard_id_11 = Shards.add("GROUPID3")

        self.__range_sharding_specification_1 = RangeShardingSpecification.add(
            self.__shard_mapping_1.shard_mapping_id,
            0,
            self.__shard_id_1.shard_id
        )
        self.__range_sharding_specification_2 = RangeShardingSpecification.add(
            self.__shard_mapping_1.shard_mapping_id,
            1001,
            self.__shard_id_2.shard_id
        )
        self.__range_sharding_specification_3 = RangeShardingSpecification.add(
            self.__shard_mapping_1.shard_mapping_id,
            2001,
            self.__shard_id_3.shard_id
        )

        self.__range_sharding_specification_4 = RangeShardingSpecification.add(
            self.__shard_mapping_2.shard_mapping_id,
            3001,
            self.__shard_id_4.shard_id
        )
        self.__range_sharding_specification_5 = RangeShardingSpecification.add(
            self.__shard_mapping_2.shard_mapping_id,
            4001,
            self.__shard_id_5.shard_id
        )

        self.__range_sharding_specification_6 = RangeShardingSpecification.add(
            self.__shard_mapping_3.shard_mapping_id,
            6001,
            self.__shard_id_6.shard_id
        )
        self.__range_sharding_specification_7 = RangeShardingSpecification.add(
            self.__shard_mapping_3.shard_mapping_id,
            7001,
            self.__shard_id_7.shard_id
        )

        self.__range_sharding_specification_8 = RangeShardingSpecification.add(
            self.__shard_mapping_4.shard_mapping_id,
            8001,
            self.__shard_id_8.shard_id
        )
        self.__range_sharding_specification_9 = RangeShardingSpecification.add(
            self.__shard_mapping_4.shard_mapping_id,
           10001,
            self.__shard_id_9.shard_id
        )

        self.__range_sharding_specification_10 = RangeShardingSpecification.add(
            self.__shard_mapping_5.shard_mapping_id,
            100,
            self.__shard_id_10.shard_id)
        self.__range_sharding_specification_11 = RangeShardingSpecification.add(
            self.__shard_mapping_5.shard_mapping_id,
            201,
            self.__shard_id_11.shard_id)
Ejemplo n.º 4
0
def _setup_shard_switch_split(shard_id, source_group_id, destination_group_id,
                              split_value, prune_limit, cmd, update_only):
    """Setup the moved shard to map to the new group.

    :param shard_id: The shard ID of the shard that needs to be moved.
    :param source_group_id: The group_id of the source shard.
    :param destn_group_id: The ID of the group to which the shard needs to
                           be moved.
    :param split_value: Indicates the value at which the range for the
                        particular shard will be split. Will be set only
                        for shard split operations.
    :param prune_limit: The number of DELETEs that should be
                        done in one batch.
    :param cmd: Indicates the type of re-sharding operation.
    :update_only: Only update the state store and skip provisioning.
    """
    #Fetch the Range sharding specification.
    range_sharding_spec, source_shard, shard_mappings, shard_mapping_defn = \
            _services_sharding.verify_and_fetch_shard(shard_id)

    #Disable the old shard
    source_shard.disable()

    #Remove the old shard.
    range_sharding_spec.remove()
    source_shard.remove()

    destination_group = Group.fetch(destination_group_id)
    if destination_group is None:
        raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
                                    (destination_group_id, ))
    destn_group_master = MySQLServer.fetch(destination_group.master)
    if destn_group_master is None:
        raise _errors.ShardingError(
            _services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
    destn_group_master.connect()

    #Make the destination group as read only to disable updates until the
    #connectors update their caches, thus avoiding inconsistency.
    destn_group_master.read_only = True

    #Add the new shards. Generate new shard IDs for the shard being
    #split and also for the shard that is created as a result of the split.
    new_shard_1 = Shards.add(source_shard.group_id, "DISABLED")
    new_shard_2 = Shards.add(destination_group_id, "DISABLED")

    #Both of the shard mappings associated with this shard_id should
    #be of the same sharding type. Hence it is safe to use one of the
    #shard mappings.
    if shard_mappings[0].type_name == "HASH":
        #In the case of a split involving a HASH sharding scheme,
        #the shard that is split gets a new shard_id, while the split
        #gets the new computed lower_bound and also a new shard id.
        #NOTE: How the shard that is split retains its lower_bound.
        HashShardingSpecification.add_hash_split(
            range_sharding_spec.shard_mapping_id, new_shard_1.shard_id,
            range_sharding_spec.lower_bound)
        HashShardingSpecification.add_hash_split(
            range_sharding_spec.shard_mapping_id, new_shard_2.shard_id,
            split_value)
    else:
        #Add the new ranges. Note that the shard being split retains
        #its lower_bound, while the new shard gets the computed,
        #lower_bound.
        RangeShardingSpecification.add(range_sharding_spec.shard_mapping_id,
                                       range_sharding_spec.lower_bound,
                                       new_shard_1.shard_id)
        RangeShardingSpecification.add(range_sharding_spec.shard_mapping_id,
                                       split_value, new_shard_2.shard_id)

    #The sleep ensures that the connector have refreshed their caches with the
    #new shards that have been added as a result of the split.
    time.sleep(_utils.TTL)

    #The source shard group master would have been marked as read only
    #during the sync. Remove the read_only flag.
    source_group = Group.fetch(source_group_id)
    if source_group is None:
        raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
                                    (source_group_id, ))

    source_group_master = MySQLServer.fetch(source_group.master)
    if source_group_master is None:
        raise _errors.ShardingError(
            _services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
    source_group_master.connect()

    #Kill all the existing connections on the servers
    source_group.kill_connections_on_servers()

    #Allow connections on the source group master
    source_group_master.read_only = False

    #Allow connections on the destination group master
    destn_group_master.read_only = False

    #Setup replication for the new group from the global server
    _group_replication.setup_group_replication \
            (shard_mapping_defn[2], destination_group_id)

    #Enable the split shards
    new_shard_1.enable()
    new_shard_2.enable()

    #Trigger changing the mappings for the shard that was copied
    if not update_only:
        _events.trigger_within_procedure(PRUNE_SHARDS, new_shard_1.shard_id,
                                         new_shard_2.shard_id, prune_limit)
Ejemplo n.º 5
0
    def setUp(self):
        """Configure the existing environment
        """
        self.__options_1 = {
            "uuid": _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
            "address": "server_1.mysql.com:3060",
            "user": MySQLInstances().user,
            "passwd": MySQLInstances().passwd,
        }
        self.__server_1 = MySQLServer(**self.__options_1)
        MySQLServer.add(self.__server_1)

        self.__options_2 = {
            "uuid": _uuid.UUID("{aa75a12a-98d1-414c-96af-9e9d4b179678}"),
            "address": "server_2.mysql.com:3060",
            "user": MySQLInstances().user,
            "passwd": MySQLInstances().passwd,
        }
        self.__server_2 = MySQLServer(**self.__options_2)
        MySQLServer.add(self.__server_2)
        self.__group_1 = Group("GROUPID1", "First description.")
        Group.add(self.__group_1)
        self.__group_1.add_server(self.__server_1)
        self.__group_1.add_server(self.__server_2)

        self.__options_3 = {
            "uuid": _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
            "address": MySQLInstances().get_address(0),
            "user": MySQLInstances().user,
            "passwd": MySQLInstances().passwd,
        }
        uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
        self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
        self.__server_3 = MySQLServer(**self.__options_3)
        MySQLServer.add(self.__server_3)
        self.__server_3.connect()
        self.__server_3.exec_stmt("DROP DATABASE IF EXISTS prune_db")
        self.__server_3.exec_stmt("CREATE DATABASE prune_db")
        self.__server_3.exec_stmt("CREATE TABLE prune_db.prune_table"
                                  "(userID INT, name VARCHAR(30))")
        self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table "
                                  "VALUES(101, 'TEST 1')")
        self.__server_3.exec_stmt("INSERT INTO prune_db.prune_table "
                                  "VALUES(202, 'TEST 2')")

        self.__options_4 = {
            "uuid": _uuid.UUID("{dd75a12a-98d1-414c-96af-9e9d4b179678}"),
            "address": "server_4.mysql.com:3060",
            "user": MySQLInstances().user,
            "passwd": MySQLInstances().passwd,
        }
        self.__server_4 = MySQLServer(**self.__options_4)
        MySQLServer.add(self.__server_4)
        self.__group_2 = Group("GROUPID2", "Second description.")
        Group.add(self.__group_2)
        self.__group_2.add_server(self.__server_3)
        self.__group_2.add_server(self.__server_4)
        tests.utils.configure_decoupled_master(self.__group_2, self.__server_3)

        self.__options_5 = {
            "uuid": _uuid.UUID("{ee75b12b-98d1-414c-96af-9e9d4b179678}"),
            "address": MySQLInstances().get_address(2),
            "user": MySQLInstances().user,
            "passwd": MySQLInstances().passwd,
        }
        uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
        self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
        self.__server_5 = MySQLServer(**self.__options_5)
        MySQLServer.add(self.__server_5)
        self.__server_5.connect()
        self.__server_5.exec_stmt("DROP DATABASE IF EXISTS prune_db")
        self.__server_5.exec_stmt("CREATE DATABASE prune_db")
        self.__server_5.exec_stmt("CREATE TABLE prune_db.prune_table"
                                  "(userID INT, name VARCHAR(30))")
        self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table "
                                  "VALUES(101, 'TEST 1')")
        self.__server_5.exec_stmt("INSERT INTO prune_db.prune_table "
                                  "VALUES(202, 'TEST 2')")

        self.__options_6 = {
            "uuid": _uuid.UUID("{ff75a12a-98d1-414c-96af-9e9d4b179678}"),
            "address": "server_6.mysql.com:3060",
            "user": MySQLInstances().user,
            "passwd": MySQLInstances().passwd,
        }
        self.__server_6 = MySQLServer(**self.__options_6)
        MySQLServer.add(self.__server_6)
        self.__group_3 = Group("GROUPID3", "Third description.")
        Group.add(self.__group_3)
        self.__group_3.add_server(self.__server_5)
        self.__group_3.add_server(self.__server_6)
        tests.utils.configure_decoupled_master(self.__group_3, self.__server_5)

        group_4 = Group("GROUPID4", "4TH description.")
        Group.add(group_4)
        group_5 = Group("GROUPID5", "5TH description.")
        Group.add(group_5)
        group_6 = Group("GROUPID6", "6TH description.")
        Group.add(group_6)
        group_7 = Group("GROUPID7", "7TH description.")
        Group.add(group_7)
        group_8 = Group("GROUPID8", "8TH description.")
        Group.add(group_8)
        group_9 = Group("GROUPID9", "9TH description.")
        Group.add(group_9)
        group_10 = Group("GROUPID10", "10TH description.")
        Group.add(group_10)
        group_11 = Group("GROUPID11", "11TH description.")
        Group.add(group_11)
        group_12 = Group("GROUPID12", "12TH description.")
        Group.add(group_12)
        group_13 = Group("GROUPID13", "13TH description.")
        Group.add(group_13)
        group_14 = Group("GROUPID14", "14TH description.")
        Group.add(group_14)

        self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
        self.assertEquals(self.__shard_mapping_list, [])
        self.__shard_mapping_id_1 = ShardMapping.define("RANGE", "GROUPID10")
        self.__shard_mapping_id_2 = ShardMapping.define("RANGE", "GROUPID11")
        self.__shard_mapping_id_3 = ShardMapping.define("RANGE", "GROUPID12")
        #Test with sharding type values in lower case
        self.__shard_mapping_id_4 = ShardMapping.define("range", "GROUPID13")
        self.__shard_mapping_id_5 = ShardMapping.define("range", "GROUPID14")

        self.__shard_mapping_1 = \
            ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID1")
        self.__shard_mapping_2 = \
            ShardMapping.add(self.__shard_mapping_id_2, "db2.t2", "userID2")
        self.__shard_mapping_3 = \
            ShardMapping.add(self.__shard_mapping_id_3, "db3.t3", "userID3")
        self.__shard_mapping_4 = \
            ShardMapping.add(self.__shard_mapping_id_4, "db4.t4", "userID4")

        self.__shard_mapping_5 = \
            ShardMapping.add(self.__shard_mapping_id_5, "prune_db.prune_table",
                             "userID")

        self.__shard_id_1 = Shards.add("GROUPID1")
        self.__shard_id_2 = Shards.add("GROUPID10")
        self.__shard_id_3 = Shards.add("GROUPID11")
        self.__shard_id_4 = Shards.add("GROUPID4")
        self.__shard_id_5 = Shards.add("GROUPID5")
        self.__shard_id_6 = Shards.add("GROUPID6")
        self.__shard_id_7 = Shards.add("GROUPID7")
        self.__shard_id_8 = Shards.add("GROUPID8")
        self.__shard_id_9 = Shards.add("GROUPID9")
        self.__shard_id_10 = Shards.add("GROUPID2")
        self.__shard_id_11 = Shards.add("GROUPID3")

        self.__range_sharding_specification_1 = RangeShardingSpecification.add(
            self.__shard_mapping_1.shard_mapping_id, 0,
            self.__shard_id_1.shard_id)
        self.__range_sharding_specification_2 = RangeShardingSpecification.add(
            self.__shard_mapping_1.shard_mapping_id, 1001,
            self.__shard_id_2.shard_id)
        self.__range_sharding_specification_3 = RangeShardingSpecification.add(
            self.__shard_mapping_1.shard_mapping_id, 2001,
            self.__shard_id_3.shard_id)

        self.__range_sharding_specification_4 = RangeShardingSpecification.add(
            self.__shard_mapping_2.shard_mapping_id, 3001,
            self.__shard_id_4.shard_id)
        self.__range_sharding_specification_5 = RangeShardingSpecification.add(
            self.__shard_mapping_2.shard_mapping_id, 4001,
            self.__shard_id_5.shard_id)

        self.__range_sharding_specification_6 = RangeShardingSpecification.add(
            self.__shard_mapping_3.shard_mapping_id, 6001,
            self.__shard_id_6.shard_id)
        self.__range_sharding_specification_7 = RangeShardingSpecification.add(
            self.__shard_mapping_3.shard_mapping_id, 7001,
            self.__shard_id_7.shard_id)

        self.__range_sharding_specification_8 = RangeShardingSpecification.add(
            self.__shard_mapping_4.shard_mapping_id, 8001,
            self.__shard_id_8.shard_id)
        self.__range_sharding_specification_9 = RangeShardingSpecification.add(
            self.__shard_mapping_4.shard_mapping_id, 10001,
            self.__shard_id_9.shard_id)

        self.__range_sharding_specification_10 = RangeShardingSpecification.add(
            self.__shard_mapping_5.shard_mapping_id, 100,
            self.__shard_id_10.shard_id)
        self.__range_sharding_specification_11 = RangeShardingSpecification.add(
            self.__shard_mapping_5.shard_mapping_id, 201,
            self.__shard_id_11.shard_id)
    def setUp(self):
        """Configure the existing environment
        """
        self.manager, self.proxy = tests.utils.setup_xmlrpc()
        self.__options_1 = {
            "uuid" :  _uuid.UUID("{bb75b12b-98d1-414c-96af-9e9d4b179678}"),
            "address"  : "server_1.mysql.com:3060",
        }
        self.__server_1 = MySQLServer(**self.__options_1)
        MySQLServer.add(self.__server_1)
        self.__options_2 = {
            "uuid" :  _uuid.UUID("{aa75a12a-98d1-414c-96af-9e9d4b179678}"),
            "address"  : "server_2.mysql.com:3060",
        }
        self.__server_2 = MySQLServer(**self.__options_2)
        MySQLServer.add(self.__server_2)
        self.__group_1 = Group("GROUPID1", "First description.")
        Group.add(self.__group_1)
        self.__group_1.add_server(self.__server_1)
        self.__group_1.add_server(self.__server_2)

        self.__options_3 = {
            "uuid" :  _uuid.UUID("{cc75b12b-98d1-414c-96af-9e9d4b179678}"),
            "address"  : MySQLInstances().get_address(0),
            "user" : MySQLInstances().user,
            "passwd" : MySQLInstances().passwd,
        }
        uuid_server3 = MySQLServer.discover_uuid(self.__options_3["address"])
        self.__options_3["uuid"] = _uuid.UUID(uuid_server3)
        self.__server_3 = MySQLServer(**self.__options_3)
        MySQLServer.add(self.__server_3)

        self.__options_4 = {
            "uuid" :  _uuid.UUID("{dd75a12a-98d1-414c-96af-9e9d4b179678}"),
            "address"  : "server_4.mysql.com:3060",
        }
        self.__server_4 = MySQLServer(**self.__options_4)
        MySQLServer.add(self.__server_4)
        self.__group_2 = Group("GROUPID2", "Second description.")
        Group.add(self.__group_2)
        self.__group_2.add_server(self.__server_3)
        self.__group_2.add_server(self.__server_4)
        tests.utils.configure_decoupled_master(self.__group_2, self.__server_3)

        self.__options_5 = {
            "uuid" :  _uuid.UUID("{ee75b12b-98d1-414c-96af-9e9d4b179678}"),
            "address"  : MySQLInstances().get_address(2),
            "user" : MySQLInstances().user,
            "passwd" : MySQLInstances().passwd,
        }
        uuid_server5 = MySQLServer.discover_uuid(self.__options_5["address"])
        self.__options_5["uuid"] = _uuid.UUID(uuid_server5)
        self.__server_5 = MySQLServer(**self.__options_5)
        MySQLServer.add(self.__server_5)

        self.__options_6 = {
            "uuid" :  _uuid.UUID("{ff75a12a-98d1-414c-96af-9e9d4b179678}"),
            "address"  : "server_6.mysql.com:3060",
        }
        self.__server_6 = MySQLServer(**self.__options_6)
        MySQLServer.add(self.__server_6)
        self.__group_3 = Group("GROUPID3", "Third description.")
        Group.add(self.__group_3)
        self.__group_3.add_server(self.__server_5)
        self.__group_3.add_server(self.__server_6)
        tests.utils.configure_decoupled_master(self.__group_3, self.__server_5)

        self.__options_1_host,  self.__options_1_port = \
            server_utils.split_host_port(self.__options_1["address"], 13001)
        self.__options_2_host,  self.__options_2_port = \
            server_utils.split_host_port(self.__options_2["address"], 13001)
        self.__options_3_host,  self.__options_3_port = \
            server_utils.split_host_port(self.__options_3["address"], 13001)
        self.__options_4_host,  self.__options_4_port = \
            server_utils.split_host_port(self.__options_4["address"], 13001)
        self.__options_5_host,  self.__options_5_port = \
            server_utils.split_host_port(self.__options_5["address"], 13001)
        self.__options_6_host,  self.__options_6_port = \
            server_utils.split_host_port(self.__options_6["address"], 13001)

        group_4 = Group("GROUPID4", "4TH description.")
        Group.add(group_4)
        group_5 = Group("GROUPID5", "5TH description.")
        Group.add(group_5)
        group_6 = Group("GROUPID6", "6TH description.")
        Group.add(group_6)
        group_7 = Group("GROUPID7", "7TH description.")
        Group.add(group_7)
        group_8 = Group("GROUPID8", "8TH description.")
        Group.add(group_8)
        group_9 = Group("GROUPID9", "9TH description.")
        Group.add(group_9)
        group_10 = Group("GROUPID10", "10TH description.")
        Group.add(group_10)
        group_11 = Group("GROUPID11", "11TH description.")
        Group.add(group_11)
        group_12 = Group("GROUPID12", "12TH description.")
        Group.add(group_12)
        group_13 = Group("GROUPID13", "13TH description.")
        Group.add(group_13)
        group_14 = Group("GROUPID14", "14TH description.")
        Group.add(group_14)

        self.__shard_mapping_list = ShardMapping.list_shard_mapping_defn()
        self.assertEquals( self.__shard_mapping_list,  [])
        self.__shard_mapping_id_1 = ShardMapping.define("RANGE", "GROUPID10")
        self.__shard_mapping_id_2 = ShardMapping.define("RANGE", "GROUPID11")
        self.__shard_mapping_id_3 = ShardMapping.define("RANGE", "GROUPID12")
        #Test with sharding type values in lower case
        self.__shard_mapping_id_4 = ShardMapping.define("range", "GROUPID13")
        self.__shard_mapping_id_5 = ShardMapping.define("range", "GROUPID14")

        self.__shard_mapping_1 = \
            ShardMapping.add(self.__shard_mapping_id_1, "db1.t1", "userID1")
        self.__shard_mapping_2 = \
            ShardMapping.add(self.__shard_mapping_id_2, "db2.t2", "userID2")
        self.__shard_mapping_3 = \
            ShardMapping.add(self.__shard_mapping_id_3, "db3.t3", "userID3")
        self.__shard_mapping_4 = \
            ShardMapping.add(self.__shard_mapping_id_4, "db4.t4", "userID4")

        self.__shard_mapping_5 = \
            ShardMapping.add(self.__shard_mapping_id_5, "prune_db.prune_table",
                             "userID")

        self.__shard_id_1 = Shards.add("GROUPID1", "ENABLED")
        self.__shard_id_2 = Shards.add("GROUPID10", "ENABLED")
        self.__shard_id_3 = Shards.add("GROUPID11", "DISABLED")
        self.__shard_id_4 = Shards.add("GROUPID4", "ENABLED")
        self.__shard_id_5 = Shards.add("GROUPID5", "ENABLED")
        self.__shard_id_6 = Shards.add("GROUPID6", "ENABLED")
        self.__shard_id_7 = Shards.add("GROUPID7", "ENABLED")
        self.__shard_id_8 = Shards.add("GROUPID8", "ENABLED")
        self.__shard_id_9 = Shards.add("GROUPID9", "ENABLED")
        self.__shard_id_10 = Shards.add("GROUPID2", "ENABLED")
        self.__shard_id_11 = Shards.add("GROUPID3", "ENABLED")

        self.__range_sharding_specification_1 = RangeShardingSpecification.add(
            self.__shard_mapping_1.shard_mapping_id,
            0,
            self.__shard_id_1.shard_id
        )
        self.__range_sharding_specification_2 = RangeShardingSpecification.add(
            self.__shard_mapping_1.shard_mapping_id,
            1001,
            self.__shard_id_2.shard_id
        )
        self.__range_sharding_specification_3 = RangeShardingSpecification.add(
            self.__shard_mapping_1.shard_mapping_id,
            2001,
            self.__shard_id_3.shard_id
        )

        self.__range_sharding_specification_4 = RangeShardingSpecification.add(
            self.__shard_mapping_2.shard_mapping_id,
            3001,
            self.__shard_id_4.shard_id
        )
        self.__range_sharding_specification_5 = RangeShardingSpecification.add(
            self.__shard_mapping_2.shard_mapping_id,
            4001,
            self.__shard_id_5.shard_id
        )

        self.__range_sharding_specification_6 = RangeShardingSpecification.add(
            self.__shard_mapping_3.shard_mapping_id,
            6001,
            self.__shard_id_6.shard_id
        )
        self.__range_sharding_specification_7 = RangeShardingSpecification.add(
            self.__shard_mapping_3.shard_mapping_id,
            7001,
            self.__shard_id_7.shard_id
        )

        self.__range_sharding_specification_8 = RangeShardingSpecification.add(
            self.__shard_mapping_4.shard_mapping_id,
            8001,
            self.__shard_id_8.shard_id
        )
        self.__range_sharding_specification_9 = RangeShardingSpecification.add(
            self.__shard_mapping_4.shard_mapping_id,
           10001,
            self.__shard_id_9.shard_id
        )

        self.__range_sharding_specification_10 = RangeShardingSpecification.add(
            self.__shard_mapping_5.shard_mapping_id,
            100,
            self.__shard_id_10.shard_id)
        self.__range_sharding_specification_11 = RangeShardingSpecification.add(
            self.__shard_mapping_5.shard_mapping_id,
            201,
            self.__shard_id_11.shard_id)

        READ_ONLY = MySQLServer.get_mode_idx(MySQLServer.READ_ONLY)
        READ_WRITE = MySQLServer.get_mode_idx(MySQLServer.READ_WRITE)

        SECONDARY = MySQLServer.get_status_idx(MySQLServer.SECONDARY)
        PRIMARY = MySQLServer.get_status_idx(MySQLServer.PRIMARY)

        self.__setofservers = [0, 0, 0,
            [[str(self.__server_1.uuid),
            'GROUPID1', self.__options_1_host,  self.__options_1_port,
            READ_ONLY, SECONDARY, 1.0],
            [str(self.__server_2.uuid),
            'GROUPID1', self.__options_2_host,  self.__options_2_port,
            READ_ONLY, SECONDARY, 1.0],
            [str(self.__server_3.uuid),
            'GROUPID2', self.__options_3_host,  self.__options_3_port,
            READ_WRITE, PRIMARY, 1.0],
            [str(self.__server_4.uuid),
            'GROUPID2', self.__options_4_host,  self.__options_4_port,
            READ_ONLY, SECONDARY, 1.0],
            [str(self.__server_5.uuid),
            'GROUPID3', self.__options_5_host,  self.__options_5_port,
            READ_WRITE, PRIMARY, 1.0],
            [str(self.__server_6.uuid),
            'GROUPID3', self.__options_6_host,  self.__options_6_port,
            READ_ONLY, SECONDARY, 1.0]]]

        self.__setofservers_1 = [0, 0, 0,
                [[str(self.__server_1.uuid),
                'GROUPID1', self.__options_1_host,  self.__options_1_port,
                READ_ONLY, SECONDARY, 1.0],
                [str(self.__server_2.uuid),
                'GROUPID1', self.__options_2_host,  self.__options_2_port,
                READ_ONLY, SECONDARY, 1.0]]]

        self.__setofservers_2 = [0, 0, 0,
                [[str(self.__server_1.uuid),
                'GROUPID1', self.__options_1_host,  self.__options_1_port,
                READ_ONLY, SECONDARY, 1.0],
                [str(self.__server_2.uuid),
                'GROUPID1', self.__options_2_host,  self.__options_2_port,
                READ_ONLY, SECONDARY, 1.0],
                [str(self.__server_3.uuid),
                'GROUPID2', self.__options_3_host,  self.__options_3_port,
                READ_WRITE, PRIMARY, 1.0],
                [str(self.__server_4.uuid),
                'GROUPID2', self.__options_4_host,  self.__options_4_port,
                READ_ONLY, SECONDARY, 1.0]]]

        self.__setofservers_3 = [0, 0, 0,
            [[str(self.__server_1.uuid),
            'GROUPID1', self.__options_1_host,  self.__options_1_port,
            READ_ONLY, SECONDARY, 1.0],
            [str(self.__server_2.uuid),
            'GROUPID1', self.__options_2_host,  self.__options_2_port,
            READ_ONLY, SECONDARY, 1.0],
            [str(self.__server_3.uuid),
            'GROUPID2', self.__options_3_host,  self.__options_3_port,
            READ_WRITE, PRIMARY, 1.0],
            [str(self.__server_4.uuid),
            'GROUPID2', self.__options_4_host,  self.__options_4_port,
            READ_ONLY, SECONDARY, 1.0],
            [str(self.__server_5.uuid),
            'GROUPID3', self.__options_5_host,  self.__options_5_port,
            READ_WRITE, PRIMARY, 1.0],
            [str(self.__server_6.uuid),
            'GROUPID3', self.__options_6_host,  self.__options_6_port,
            READ_ONLY, SECONDARY, 1.0]]]

        self.__setoftables = [0, 0, 0, [['db1', 't1', 'userID1', '1'],
                              ['db2', 't2', 'userID2', '2'],
                              ['db3', 't3', 'userID3', '3'],
                              ['db4', 't4', 'userID4', '4'],
                              ['prune_db', 'prune_table', 'userID', '5']]]
        self.__setoftables_1 = [0, 0, 0, [['db1', 't1', 'userID1', '1']]]
        self.__setoftables_2 = [0, 0, 0, [['db1', 't1', 'userID1', '1'],
                                 ['db2', 't2', 'userID2', '2']]]
        self.__setoftables_3 = [0, 0, 0, [['db1', 't1', 'userID1', '1'],
                                ['db2', 't2', 'userID2', '2'],
                                ['db3', 't3', 'userID3', '3']]]
        self.__setofshardmaps = [0, 0, 0, [['1', 'RANGE', 'GROUPID10'],
                                 ['2', 'RANGE', 'GROUPID11'],
                                 ['3', 'RANGE', 'GROUPID12'],
                                 ['4', 'RANGE', 'GROUPID13'],
                                 ['5', 'RANGE', 'GROUPID14']]]
        self.__setofshardmaps_1 = [0, 0, 0, [['1', 'RANGE', 'GROUPID10']]]
        self.__setofshardmaps_2 = [0, 0, 0, [['1', 'RANGE', 'GROUPID10'],
                                 ['2', 'RANGE', 'GROUPID11']]]
        self.__setofshardmaps_3 = [0, 0, 0, [['1', 'RANGE', 'GROUPID10'],
                                 ['2', 'RANGE', 'GROUPID11'],
                                 ['3', 'RANGE', 'GROUPID12']]]
        self.__setofshardindexes = [0, 0, 0, [['0', '1', '1', 'GROUPID1'],
                                    ['1001', '1', '2', 'GROUPID10'],
                                    ['3001', '2', '4', 'GROUPID4'],
                                    ['4001', '2', '5', 'GROUPID5'],
                                    ['6001', '3', '6', 'GROUPID6'],
                                    ['7001', '3', '7', 'GROUPID7'],
                                    ['8001', '4', '8', 'GROUPID8'],
                                    ['10001', '4', '9', 'GROUPID9'],
                                    ['100', '5', '10', 'GROUPID2'],
                                    ['201', '5', '11', 'GROUPID3']]]
        self.__setofshardindexes_1 = [0, 0, 0, [['0', '1', '1', 'GROUPID1'],
                                      ['1001', '1', '2', 'GROUPID10']]]
        self.__setofshardindexes_3 = [0, 0, 0, [['0', '1', '1', 'GROUPID1'],
                                      ['1001', '1', '2', 'GROUPID10'],
                                      ['3001', '2', '4', 'GROUPID4'],
                                      ['4001', '2', '5', 'GROUPID5'],
                                          ['6001', '3', '6', 'GROUPID6'],
                                          ['7001', '3', '7', 'GROUPID7']]]
        self.__setofshardindexes_5 = [0, 0, 0, [['0', '1', '1', 'GROUPID1'],
                                      ['1001', '1', '2', 'GROUPID10'],
                                      ['3001', '2', '4', 'GROUPID4'],
                                      ['4001', '2', '5', 'GROUPID5'],
                                      ['6001', '3', '6', 'GROUPID6'],
                                      ['7001', '3', '7', 'GROUPID7'],
                                      ['8001', '4', '8', 'GROUPID8'],
                                      ['10001', '4', '9', 'GROUPID9'],
                                      ['100', '5', '10', 'GROUPID2'],
                                      ['201', '5', '11', 'GROUPID3']]]
        self.__shardinginformation_1 = [0, 0, 0, [['db1', 't1', 'userID1', '0',
                                        '1', 'RANGE', 'GROUPID1', 'GROUPID10'],
                                        ['db1', 't1', 'userID1', '1001',
                                        '2', 'RANGE', 'GROUPID10', 'GROUPID10']]]
        self.__shardinginformation_2 = [0, 0, 0, [['db1', 't1', 'userID1', '0',
                                        '1', 'RANGE', 'GROUPID1', 'GROUPID10'],
                                        ['db1', 't1', 'userID1', '1001',
                                        '2', 'RANGE', 'GROUPID10', 'GROUPID10'],
                                        ['db2', 't2', 'userID2', '3001',
                                        '4', 'RANGE', 'GROUPID4', 'GROUPID11'],
                                        ['db2', 't2', 'userID2', '4001',
                                        '5', 'RANGE', 'GROUPID5', 'GROUPID11']]]
        self.__shardinginformation_3 = [0, 0, 0, [['db1', 't1', 'userID1', '0',
                                        '1', 'RANGE', 'GROUPID1', 'GROUPID10'],
                                        ['db1', 't1', 'userID1', '1001',
                                        '2', 'RANGE', 'GROUPID10', 'GROUPID10'],
                                        ['db2', 't2', 'userID2', '3001',
                                        '4','RANGE', 'GROUPID4', 'GROUPID11'],
                                        ['db2', 't2', 'userID2', '4001',
                                        '5', 'RANGE', 'GROUPID5', 'GROUPID11'],
                                        ['db3', 't3', 'userID3', '6001',
                                        '6', 'RANGE', 'GROUPID6', 'GROUPID12'],
                                        ['db3', 't3', 'userID3', '7001',
                                        '7', 'RANGE', 'GROUPID7', 'GROUPID12']]]
Ejemplo n.º 7
0
def _add_shard(shard_mapping_id, groupid_lb_list, state):
    """Add the RANGE shard specification. This represents a single instance
    of a shard specification that maps a key RANGE to a server.

    :param shard_mapping_id: The unique identification for a shard mapping.
    :param groupid_lb_list: The list of group_id, lower_bounds pairs in the
                        format, group_id/lower_bound, group_id/lower_bound... .
    :param state: Indicates whether a given shard is ENABLED or DISABLED

    :return: True if the add succeeded.
                False otherwise.
    :raises: ShardingError If the group on which the shard is being
                           created does not exist,
                           If the shard_mapping_id is not found,
                           If adding the shard definition fails,
                           If the state of the shard is an invalid
                           value,
                           If the range definition is invalid.
    """
    shard_mapping = ShardMapping.fetch_shard_mapping_defn(shard_mapping_id)
    if shard_mapping is None:
        raise _errors.ShardingError(SHARD_MAPPING_NOT_FOUND % \
                                                    (shard_mapping_id,  ))

    schema_type = shard_mapping[1]

    if len(RangeShardingSpecification.list(shard_mapping_id)) != 0:
        raise _errors.ShardingError(SHARDS_ALREADY_EXIST)

    group_id_list, lower_bound_list =\
        _utils.get_group_lower_bound_list(groupid_lb_list)

    if (len(group_id_list) != len(lower_bound_list)) and\
        schema_type == "RANGE":
        raise _errors.ShardingError(LOWER_BOUND_GROUP_ID_COUNT_MISMATCH)

    if len(lower_bound_list) != 0 and schema_type == "HASH":
        raise _errors.ShardingError(LOWER_BOUND_AUTO_GENERATED)

    if schema_type == "RANGE":
        for lower_bound in lower_bound_list:
            if(not SHARDING_DATATYPE_HANDLER[schema_type].\
                        is_valid_lower_bound(lower_bound)):
                raise _errors.ShardingError(
                                INVALID_LOWER_BOUND_VALUE % (lower_bound, ))

    state = state.upper()
    if state not in Shards.VALID_SHARD_STATES:
        raise _errors.ShardingError(INVALID_SHARD_STATE % (state,  ))

    for index, group_id in enumerate(group_id_list):
        shard = Shards.add(group_id, state)

        shard_id = shard.shard_id

        if schema_type == "RANGE":
            range_sharding_specification = RangeShardingSpecification.add(
                                                shard_mapping_id,
                                                lower_bound_list[index],
                                                shard_id
                                            )
            _LOGGER.debug(
                "Added Shard (map id = %s, lower bound = %s, id = %s).",
                range_sharding_specification.shard_mapping_id,
                range_sharding_specification.lower_bound,
                range_sharding_specification.shard_id
            )
        elif schema_type == "HASH":
            HashShardingSpecification.add(
                shard_mapping_id,
                shard_id
            )
            _LOGGER.debug(
                "Added Shard (map id = %s, id = %s).",
                shard_mapping_id,
                shard_id
            )
        else:
            raise _errors.ShardingError(INVALID_SHARDING_TYPE % (schema_type, ))

        #If the shard is added in a DISABLED state  do not setup replication
        #with the primary of the global group. Basically setup replication only
        #if the shard is ENABLED.
        if state == "ENABLED":
            _setup_shard_group_replication(shard_id)