def _backup_source_shard(shard_id, source_group_id, destn_group_id,
mysqldump_binary, mysqlclient_binary, split_value,
config_file, prune_limit, cmd, update_only):
"""Backup the source shard.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param mysqldump_binary: The fully qualified mysqldump binary.
:param mysqlclient_binary: The fully qualified mysql client binary.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param config_file: The complete path to the fabric configuration file.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation (move, split)
:update_only: Only update the state store and skip provisioning.
"""
source_group = Group.fetch(source_group_id)
move_source_server = _services_utils.fetch_backup_server(source_group)
#Do the backup of the group hosting the source shard.
backup_image = _backup.MySQLDump.backup(move_source_server, config_file,
mysqldump_binary)
#Change the master for the server that is master of the group which hosts
#the destination shard.
_events.trigger_within_procedure(RESTORE_SHARD_BACKUP, shard_id,
source_group_id, destn_group_id,
mysqlclient_binary, backup_image.path,
split_value, config_file, prune_limit,
cmd)
def execute(self, group_id, destn_address, server_id=None, timeout=None,
synchronous=True):
"""Clone the objects of a given server into a destination server.
:param group_id: The ID of the source group.
:param destn_address: The address of the destination MySQL Server.
:param source_id: The address or UUID of the source MySQL Server.
:param timeout: Time in seconds after which an error is reported
if the destination server is unreachable.
:param synchronous: Whether one should wait until the execution
finishes or not.
"""
# If the destination server is already part of a Fabric Group, raise
# an error
destn_server_uuid = _lookup_uuid(destn_address, timeout)
_check_server_not_in_any_group(destn_server_uuid)
host, port = split_host_port(
destn_address, _backup.MySQLDump.MYSQL_DEFAULT_PORT
)
# Fetch information on backup and restore programs.
config_file = self.config.config_file if self.config.config_file else ""
mysqldump_binary = _utils.read_config_value(
self.config,
'sharding',
'mysqldump_program'
)
mysqlclient_binary = _utils.read_config_value(
self.config,
'sharding',
'mysqlclient_program'
)
if not _utils.is_valid_binary(mysqldump_binary):
raise _errors.ServerError(MYSQLDUMP_NOT_FOUND % mysqldump_binary)
if not _utils.is_valid_binary(mysqlclient_binary):
raise _errors.ServerError(MYSQLCLIENT_NOT_FOUND % mysqlclient_binary)
# Fetch a reference to source server.
if server_id:
server = _retrieve_server(server_id, group_id)
else:
group = _retrieve_group(group_id)
server = _utils.fetch_backup_server(group)
# Schedule the clone operation through the executor.
procedures = _events.trigger(
BACKUP_SERVER,
self.get_lockable_objects(),
str(server.uuid),
host,
port,
mysqldump_binary,
mysqlclient_binary,
config_file
)
return self.wait_for_procedures(procedures, synchronous)
def _backup_source_shard(shard_id, source_group_id, destn_group_id,
split_value, prune_limit, cmd, update_only):
"""Backup the source shard.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation (move, split)
:update_only: Only update the state store and skip provisioning.
"""
backup_user = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_user'
)
backup_passwd = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_password'
)
mysqldump_binary = _services_utils.read_config_value(
_config.global_config,
'sharding',
'mysqldump_program'
)
source_group = Group.fetch(source_group_id)
move_source_server = _services_utils.fetch_backup_server(source_group)
#Do the backup of the group hosting the source shard.
backup_image = _backup.MySQLDump.backup(
move_source_server,
backup_user, backup_passwd,
mysqldump_binary
)
#Change the master for the server that is master of the group which hosts
#the destination shard.
_events.trigger_within_procedure(
RESTORE_SHARD_BACKUP,
shard_id,
source_group_id,
destn_group_id,
backup_image.path,
split_value,
prune_limit,
cmd
)
def _backup_source_shard(shard_id, source_group_id, destn_group_id,
split_value, prune_limit, cmd, update_only):
"""Backup the source shard.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation (move, split)
:update_only: Only update the state store and skip provisioning.
"""
backup_user = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_user'
)
backup_passwd = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_password'
)
mysqldump_binary = _services_utils.read_config_value(
_config.global_config,
'sharding',
'mysqldump_program'
)
source_group = Group.fetch(source_group_id)
move_source_server = _services_utils.fetch_backup_server(source_group)
#Do the backup of the group hosting the source shard.
backup_image = _backup.MySQLDump.backup(
move_source_server,
backup_user, backup_passwd,
mysqldump_binary
)
#Change the master for the server that is master of the group which hosts
#the destination shard.
_events.trigger_within_procedure(
RESTORE_SHARD_BACKUP,
shard_id,
source_group_id,
destn_group_id,
backup_image.path,
split_value,
prune_limit,
cmd
)
def execute(self,
group_id,
destn_address,
server_id=None,
timeout=None,
synchronous=True):
"""Clone the objects of a given server into a destination server.
:param group_id: The ID of the source group.
:param destn_address: The address of the destination MySQL Server.
:param source_id: The address or UUID of the source MySQL Server.
:param timeout: Time in seconds after which an error is reported
if the destination server is unreachable.
:param synchronous: Whether one should wait until the execution
finishes or not.
"""
# If the destination server is already part of a Fabric Group, raise
# an error
destn_server_uuid = _lookup_uuid(destn_address, timeout)
_check_server_not_in_any_group(destn_server_uuid)
host, port = split_host_port(destn_address,
_backup.MySQLDump.MYSQL_DEFAULT_PORT)
# Fetch information on backup and restore programs.
config_file = self.config.config_file if self.config.config_file else ""
mysqldump_binary = _utils.read_config_value(self.config, 'sharding',
'mysqldump_program')
mysqlclient_binary = _utils.read_config_value(self.config, 'sharding',
'mysqlclient_program')
if not _utils.is_valid_binary(mysqldump_binary):
raise _errors.ServerError(MYSQLDUMP_NOT_FOUND % mysqldump_binary)
if not _utils.is_valid_binary(mysqlclient_binary):
raise _errors.ServerError(MYSQLCLIENT_NOT_FOUND %
mysqlclient_binary)
# Fetch a reference to source server.
if server_id:
server = _retrieve_server(server_id, group_id)
else:
group = _retrieve_group(group_id)
server = _utils.fetch_backup_server(group)
# Schedule the clone operation through the executor.
procedures = _events.trigger(BACKUP_SERVER,
self.get_lockable_objects(),
str(server.uuid), host, port,
mysqldump_binary, mysqlclient_binary,
config_file)
return self.wait_for_procedures(procedures, synchronous)
def _backup_source_shard(shard_id, source_group_id, destn_group_id,
mysqldump_binary, mysqlclient_binary, split_value,
config_file, prune_limit, cmd, update_only):
"""Backup the source shard.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param mysqldump_binary: The fully qualified mysqldump binary.
:param mysqlclient_binary: The fully qualified mysql client binary.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param config_file: The complete path to the fabric configuration file.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation (move, split)
:update_only: Only update the state store and skip provisioning.
"""
source_group = Group.fetch(source_group_id)
move_source_server = _services_utils.fetch_backup_server(source_group)
#Do the backup of the group hosting the source shard.
backup_image = _backup.MySQLDump.backup(
move_source_server,
config_file,
mysqldump_binary
)
#Change the master for the server that is master of the group which hosts
#the destination shard.
_events.trigger_within_procedure(
RESTORE_SHARD_BACKUP,
shard_id,
source_group_id,
destn_group_id,
mysqlclient_binary,
backup_image.path,
split_value,
config_file,
prune_limit,
cmd
)
def execute(self, group_id, destn_address, server_uuid=None,
synchronous=True):
"""Clone the objects of a given server into a destination server.
:param group_id: The ID of the source group.
:param destn_address: The address of the MySQL Server to which
the clone needs to happen.
:param server_uuid: The UUID of the source MySQL Server.
:param synchronous: Whether one should wait until the execution
finishes or not.
"""
#If the destination server is already part of a Fabric Group, raise
#an error
if destn_address:
destn_server_uuid = _server.MySQLServer.\
discover_uuid(destn_address)
destn_server = _server.MySQLServer.fetch(destn_server_uuid)
#we should check for both the presence of the server object
#and its associated group ID. Checking its association with
#a group ID would verify that a server that is part of Fabric
#but is not part of a group can also be cloned into.
if destn_server and destn_server.group_id:
raise _errors.ServerError("The Destination server is already "\
"part of Group (%s)" % (destn_server.group_id,))
config_file = self.config.config_file if self.config.config_file else ""
mysqldump_binary = _utils.read_config_value(
self.config,
'sharding',
'mysqldump_program'
)
mysqlclient_binary = _utils.read_config_value(
self.config,
'sharding',
'mysqlclient_program'
)
if not _utils.is_valid_binary(mysqldump_binary):
raise _errors.ServerError(MYSQLDUMP_NOT_FOUND % mysqldump_binary)
if not _utils.is_valid_binary(mysqlclient_binary):
raise _errors.ServerError(MYSQLCLIENT_NOT_FOUND % mysqlclient_binary)
if server_uuid:
server = _server.MySQLServer.fetch(server_uuid)
if group_id != server.group_id:
raise _errors.ServerError("The server %s was not found in "\
"group %s" % (server_uuid, group_id, ))
elif not server_uuid:
group = _server.Group.fetch(group_id)
server = _utils.fetch_backup_server(group)
server_uuid = str(server.uuid)
host, port = _server_utils.split_host_port(
destn_address,
_backup.MySQLDump.MYSQL_DEFAULT_PORT
)
procedures = _events.trigger(
BACKUP_SERVER,
self.get_lockable_objects(),
server_uuid,
host,
port,
mysqldump_binary,
mysqlclient_binary,
config_file
)
return self.wait_for_procedures(procedures, synchronous)
def _check_shard_information(shard_id, destn_group_id,
split_value, prune_limit, cmd, update_only):
"""Verify the sharding information before starting a re-sharding operation.
:param shard_id: The destination shard ID.
:param destn_group_id: The Destination group ID.
:param split_value: The point at which the sharding definition
should be split.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates if it is a split or a move being executed.
:param update_only: If the operation is a update only operation.
"""
backup_user = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_user'
)
backup_passwd = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_password'
)
restore_user = _services_utils.read_config_value(
_config.global_config,
'servers',
'restore_user'
)
restore_passwd = _services_utils.read_config_value(
_config.global_config,
'servers',
'restore_password'
)
mysqldump_binary = _services_utils.read_config_value(
_config.global_config,
'sharding',
'mysqldump_program'
)
mysqlclient_binary = _services_utils.read_config_value(
_config.global_config,
'sharding',
'mysqlclient_program'
)
if not _services_utils.is_valid_binary(mysqldump_binary):
raise _errors.ShardingError(
_services_sharding.MYSQLDUMP_NOT_FOUND % mysqldump_binary)
if not _services_utils.is_valid_binary(mysqlclient_binary):
raise _errors.ShardingError(
_services_sharding.MYSQLCLIENT_NOT_FOUND % mysqlclient_binary)
if cmd == "SPLIT":
range_sharding_spec, _, shard_mappings, _ = \
_services_sharding.verify_and_fetch_shard(shard_id)
upper_bound = \
SHARDING_SPECIFICATION_HANDLER[shard_mappings[0].type_name].\
get_upper_bound(
range_sharding_spec.lower_bound,
range_sharding_spec.shard_mapping_id,
shard_mappings[0].type_name
)
#If the underlying sharding scheme is a HASH. When a shard is split,
#all the tables that are part of the shard, have the same sharding
#scheme. All the shard mappings associated with this shard_id will be
#of the same sharding type. Hence it is safe to use one of the shard
#mappings.
if shard_mappings[0].type_name == "HASH":
if split_value is not None:
raise _errors.ShardingError(
_services_sharding.NO_LOWER_BOUND_FOR_HASH_SHARDING
)
if upper_bound is None:
#While splitting a range, retrieve the next upper bound and
#find the mid-point, in the case where the next upper_bound
#is unavailable pick the maximum value in the set of values in
#the shard.
upper_bound = HashShardingSpecification.fetch_max_key(shard_id)
#Calculate the split value.
split_value = \
SHARDING_DATATYPE_HANDLER[shard_mappings[0].type_name].\
split_value(
range_sharding_spec.lower_bound,
upper_bound
)
elif split_value is not None:
if not (SHARDING_DATATYPE_HANDLER[shard_mappings[0].type_name].\
is_valid_split_value(
split_value, range_sharding_spec.lower_bound,
upper_bound
)
):
raise _errors.ShardingError(
_services_sharding.INVALID_LOWER_BOUND_VALUE %
(split_value, )
)
elif split_value is None:
raise _errors.ShardingError(
_services_sharding.SPLIT_VALUE_NOT_DEFINED
)
#Ensure that the group does not already contain a shard.
if Shards.lookup_shard_id(destn_group_id) is not None:
raise _errors.ShardingError(
_services_sharding.SHARD_MOVE_DESTINATION_NOT_EMPTY %
(destn_group_id, )
)
#Fetch the group information for the source shard that
#needs to be moved.
source_shard = Shards.fetch(shard_id)
if source_shard is None:
raise _errors.ShardingError(
_services_sharding.SHARD_NOT_FOUND % (shard_id, ))
#Fetch the group_id and the group that hosts the source shard.
source_group_id = source_shard.group_id
destn_group = Group.fetch(destn_group_id)
if destn_group is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_NOT_FOUND %
(destn_group_id, ))
if not update_only:
# Check if the source server has backup privileges.
source_group = Group.fetch(source_group_id)
server = _services_utils.fetch_backup_server(source_group)
server.user = backup_user
server.passwd = backup_passwd
_backup.MySQLDump.check_backup_privileges(server)
# Check if the destination server has restore privileges.
destination_group = Group.fetch(destn_group_id)
server = MySQLServer.fetch(destination_group.master)
server.user = restore_user
server.passwd = restore_passwd
_backup.MySQLDump.check_restore_privileges(server)
_events.trigger_within_procedure(
BACKUP_SOURCE_SHARD, shard_id, source_group_id, destn_group_id,
split_value, prune_limit, cmd, update_only
)
else:
_events.trigger_within_procedure(
SETUP_RESHARDING_SWITCH, shard_id, source_group_id, destn_group_id,
split_value, prune_limit, cmd, update_only
)
def _check_shard_information(shard_id, destn_group_id,
split_value, prune_limit, cmd, update_only):
"""Verify the sharding information before starting a re-sharding operation.
:param shard_id: The destination shard ID.
:param destn_group_id: The Destination group ID.
:param split_value: The point at which the sharding definition
should be split.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates if it is a split or a move being executed.
:param update_only: If the operation is a update only operation.
"""
backup_user = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_user'
)
backup_passwd = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_password'
)
restore_user = _services_utils.read_config_value(
_config.global_config,
'servers',
'restore_user'
)
restore_passwd = _services_utils.read_config_value(
_config.global_config,
'servers',
'restore_password'
)
mysqldump_binary = _services_utils.read_config_value(
_config.global_config,
'sharding',
'mysqldump_program'
)
mysqlclient_binary = _services_utils.read_config_value(
_config.global_config,
'sharding',
'mysqlclient_program'
)
if not _services_utils.is_valid_binary(mysqldump_binary):
raise _errors.ShardingError(
_services_sharding.MYSQLDUMP_NOT_FOUND % mysqldump_binary)
if not _services_utils.is_valid_binary(mysqlclient_binary):
raise _errors.ShardingError(
_services_sharding.MYSQLCLIENT_NOT_FOUND % mysqlclient_binary)
if cmd == "SPLIT":
range_sharding_spec, _, shard_mappings, _ = \
_services_sharding.verify_and_fetch_shard(shard_id)
upper_bound = \
SHARDING_SPECIFICATION_HANDLER[shard_mappings[0].type_name].\
get_upper_bound(
range_sharding_spec.lower_bound,
range_sharding_spec.shard_mapping_id,
shard_mappings[0].type_name
)
#If the underlying sharding scheme is a HASH. When a shard is split,
#all the tables that are part of the shard, have the same sharding
#scheme. All the shard mappings associated with this shard_id will be
#of the same sharding type. Hence it is safe to use one of the shard
#mappings.
if shard_mappings[0].type_name == "HASH":
if split_value is not None:
raise _errors.ShardingError(
_services_sharding.NO_LOWER_BOUND_FOR_HASH_SHARDING
)
if upper_bound is None:
#While splitting a range, retrieve the next upper bound and
#find the mid-point, in the case where the next upper_bound
#is unavailable pick the maximum value in the set of values in
#the shard.
upper_bound = HashShardingSpecification.fetch_max_key(shard_id)
#Calculate the split value.
split_value = \
SHARDING_DATATYPE_HANDLER[shard_mappings[0].type_name].\
split_value(
range_sharding_spec.lower_bound,
upper_bound
)
elif split_value is not None:
if not (SHARDING_DATATYPE_HANDLER[shard_mappings[0].type_name].\
is_valid_split_value(
split_value, range_sharding_spec.lower_bound,
upper_bound
)
):
raise _errors.ShardingError(
_services_sharding.INVALID_LOWER_BOUND_VALUE %
(split_value, )
)
elif split_value is None:
raise _errors.ShardingError(
_services_sharding.SPLIT_VALUE_NOT_DEFINED
)
#Ensure that the group does not already contain a shard.
if Shards.lookup_shard_id(destn_group_id) is not None:
raise _errors.ShardingError(
_services_sharding.SHARD_MOVE_DESTINATION_NOT_EMPTY %
(destn_group_id, )
)
#Fetch the group information for the source shard that
#needs to be moved.
source_shard = Shards.fetch(shard_id)
if source_shard is None:
raise _errors.ShardingError(
_services_sharding.SHARD_NOT_FOUND % (shard_id, ))
#Fetch the group_id and the group that hosts the source shard.
source_group_id = source_shard.group_id
destn_group = Group.fetch(destn_group_id)
if destn_group is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_NOT_FOUND %
(destn_group_id, ))
if not update_only:
# Check if the source server has backup privileges.
source_group = Group.fetch(source_group_id)
server = _services_utils.fetch_backup_server(source_group)
server.user = backup_user
server.passwd = backup_passwd
_backup.MySQLDump.check_backup_privileges(server)
# Check if the destination server has restore privileges.
destination_group = Group.fetch(destn_group_id)
server = MySQLServer.fetch(destination_group.master)
server.user = restore_user
server.passwd = restore_passwd
_backup.MySQLDump.check_restore_privileges(server)
_events.trigger_within_procedure(
BACKUP_SOURCE_SHARD, shard_id, source_group_id, destn_group_id,
split_value, prune_limit, cmd, update_only
)
else:
_events.trigger_within_procedure(
SETUP_RESHARDING_SWITCH, shard_id, source_group_id, destn_group_id,
split_value, prune_limit, cmd, update_only
)