def _backup_source_shard(shard_id, source_group_id, destn_group_id,
mysqldump_binary, mysqlclient_binary, split_value,
config_file, prune_limit, cmd, update_only):
"""Backup the source shard.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param mysqldump_binary: The fully qualified mysqldump binary.
:param mysqlclient_binary: The fully qualified mysql client binary.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param config_file: The complete path to the fabric configuration file.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation (move, split)
:update_only: Only update the state store and skip provisioning.
"""
source_group = Group.fetch(source_group_id)
move_source_server = _services_utils.fetch_backup_server(source_group)
#Do the backup of the group hosting the source shard.
backup_image = _backup.MySQLDump.backup(move_source_server, config_file,
mysqldump_binary)
#Change the master for the server that is master of the group which hosts
#the destination shard.
_events.trigger_within_procedure(RESTORE_SHARD_BACKUP, shard_id,
source_group_id, destn_group_id,
mysqlclient_binary, backup_image.path,
split_value, config_file, prune_limit,
cmd)
def _block_write_demote(group_id, update_only):
"""Block and disable write access to the current master.
"""
group = _server.Group.fetch(group_id)
if not group:
raise _errors.GroupError("Group (%s) does not exist." % (group_id, ))
if not group.master:
raise _errors.GroupError("Group (%s) does not have a master." %
(group_id, ))
master = _server.MySQLServer.fetch(group.master)
assert(master.status in \
(_server.MySQLServer.PRIMARY, _server.MySQLServer.FAULTY)
)
if master.status == _server.MySQLServer.PRIMARY:
master.connect()
master.mode = _server.MySQLServer.READ_ONLY
master.status = _server.MySQLServer.SECONDARY
_utils.set_read_only(master, True)
if not update_only:
_events.trigger_within_procedure(
WAIT_SLAVES_DEMOTE, group_id, str(master.uuid)
)
_set_group_master_replication(group, None, update_only)
def _restore_shard_backup(shard_id, source_group_id, destn_group_id,
backup_image, split_value, prune_limit, cmd):
"""Restore the backup on the destination Group.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param backup_image: The destination file that contains the backup
of the source shard.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation
"""
restore_user = _services_utils.read_config_value(
_config.global_config,
'servers',
'restore_user'
)
restore_passwd = _services_utils.read_config_value(
_config.global_config,
'servers',
'restore_password'
)
mysqlclient_binary = _services_utils.read_config_value(
_config.global_config,
'sharding',
'mysqlclient_program'
)
destn_group = Group.fetch(destn_group_id)
if destn_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(destn_group_id, ))
#Build a backup image that will be used for restoring
bk_img = _backup.BackupImage(backup_image)
for destn_group_server in destn_group.servers():
destn_group_server.connect()
_backup.MySQLDump.restore_fabric_server(
destn_group_server,
restore_user, restore_passwd,
bk_img,
mysqlclient_binary
)
#Setup sync between the source and the destination groups.
_events.trigger_within_procedure(
SETUP_REPLICATION,
shard_id,
source_group_id,
destn_group_id,
split_value,
prune_limit,
cmd
)
def _block_write_switch(group_id, master_uuid, slave_uuid):
"""Block and disable write access to the current master.
"""
_do_block_write_master(group_id, master_uuid)
_events.trigger_within_procedure(WAIT_SLAVES_SWITCH, group_id,
master_uuid, slave_uuid
)
def _set_server_status_spare(server, update_only):
"""Set server's status to spare. If the server has faulty or
configuring status, it might happen that it is not properly
configured. So the server's status is temporarily set to
configuring and the proper configuration actions are taken
in the next step.
"""
allowed_status = [
_server.MySQLServer.SECONDARY, _server.MySQLServer.FAULTY,
_server.MySQLServer.CONFIGURING
]
status = _server.MySQLServer.SPARE
mode = _server.MySQLServer.OFFLINE
previous_status = server.status
if previous_status in \
(_server.MySQLServer.FAULTY, _server.MySQLServer.CONFIGURING):
if server.is_alive():
_events.trigger_within_procedure(
CONFIGURE_FAULTY_SERVER, str(server.uuid), previous_status,
update_only
)
status = _server.MySQLServer.CONFIGURING
_do_set_status(server, allowed_status, status, mode, update_only)
def _check_candidate_fail(group_id, slave_id):
"""Check if the candidate has all the prerequisites to become the new
master.
"""
allowed_status = (_server.MySQLServer.SECONDARY, _server.MySQLServer.SPARE)
group = _server.Group.fetch(group_id)
slave = _retrieve_server(slave_id, group_id)
slave.connect()
if group.master == slave.uuid:
raise _errors.ServerError("Candidate slave (%s) is already master." %
(slave_id, ))
master_issues, why_master_issues = _replication.check_master_issues(slave)
if master_issues:
raise _errors.ServerError("Server (%s) is not a valid candidate slave "
"due to the following reason(s): (%s)." %
(slave.uuid, why_master_issues))
if slave.status not in allowed_status:
raise _errors.ServerError("Server (%s) is faulty." % (slave_id, ))
_events.trigger_within_procedure(WAIT_SLAVE_FAIL, group_id,
str(slave.uuid))
def _setup_replication(shard_id, source_group_id, destn_group_id, split_value,
prune_limit, cmd):
"""Setup replication between the source and the destination groups and
ensure that they are in sync.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation
"""
source_group = Group.fetch(source_group_id)
if source_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(source_group_id, ))
destination_group = Group.fetch(destn_group_id)
if destination_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(destn_group_id, ))
master = MySQLServer.fetch(source_group.master)
if master is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
master.connect()
slave = MySQLServer.fetch(destination_group.master)
if slave is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
slave.connect()
#Stop and reset any slave that might be running on the slave server.
_utils.set_offline_mode(slave, True) ### TODO: if forced offline_mode
_replication.stop_slave(slave, wait=True)
_replication.reset_slave(slave, clean=True)
#Change the master to the shard group master.
_replication.switch_master(slave, master, master.repl_user, master.repl_pass)
#Start the slave so that syncing of the data begins
_replication.start_slave(slave, wait=True)
_utils.set_offline_mode(slave, False) ### TODO: if forced offline_mode
#Setup sync between the source and the destination groups.
_events.trigger_within_procedure(
SETUP_SYNC,
shard_id,
source_group_id,
destn_group_id,
split_value,
prune_limit,
cmd
)
def _restore_shard_backup(shard_id, source_group_id, destn_group_id,
backup_image, split_value, prune_limit, cmd):
"""Restore the backup on the destination Group.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param backup_image: The destination file that contains the backup
of the source shard.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation
"""
restore_user = _services_utils.read_config_value(
_config.global_config,
'servers',
'restore_user'
)
restore_passwd = _services_utils.read_config_value(
_config.global_config,
'servers',
'restore_password'
)
mysqlclient_binary = _services_utils.read_config_value(
_config.global_config,
'sharding',
'mysqlclient_program'
)
destn_group = Group.fetch(destn_group_id)
if destn_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(destn_group_id, ))
#Build a backup image that will be used for restoring
bk_img = _backup.BackupImage(backup_image)
for destn_group_server in destn_group.servers():
destn_group_server.connect()
_backup.MySQLDump.restore_fabric_server(
destn_group_server,
restore_user, restore_passwd,
bk_img,
mysqlclient_binary
)
#Setup sync between the source and the destination groups.
_events.trigger_within_procedure(
SETUP_REPLICATION,
shard_id,
source_group_id,
destn_group_id,
split_value,
prune_limit,
cmd
)
def _check_candidate_fail(group_id, slave_id):
"""Check if the candidate has all the prerequisites to become the new
master.
"""
allowed_status = (_server.MySQLServer.SECONDARY, _server.MySQLServer.SPARE)
group = _server.Group.fetch(group_id)
slave = _retrieve_server(slave_id, group_id)
slave.connect()
if group.master == slave.uuid:
raise _errors.ServerError(
"Candidate slave (%s) is already master." % (slave_id, )
)
master_issues = _replication.check_master_issues(slave)
if master_issues:
raise _errors.ServerError(
"Server (%s) is not a valid candidate slave "
"due to the following reason(s): (%s)." %
(slave.uuid, master_issues)
)
if slave.status not in allowed_status:
raise _errors.ServerError("Server (%s) is faulty." % (slave_id, ))
_events.trigger_within_procedure(WAIT_SLAVE_FAIL, group_id, str(slave.uuid))
def _block_write_demote(group_id, update_only):
"""Block and disable write access to the current master.
"""
group = _server.Group.fetch(group_id)
if not group:
raise _errors.GroupError("Group (%s) does not exist." % (group_id, ))
if not group.master:
raise _errors.GroupError("Group (%s) does not have a master." %
(group_id, ))
master = _server.MySQLServer.fetch(group.master)
assert(master.status in \
(_server.MySQLServer.PRIMARY, _server.MySQLServer.FAULTY)
)
if master.status == _server.MySQLServer.PRIMARY:
master.connect()
master.mode = _server.MySQLServer.READ_ONLY
master.status = _server.MySQLServer.SECONDARY
_utils.set_read_only(master, True)
if not update_only:
_events.trigger_within_procedure(WAIT_SLAVES_DEMOTE, group_id,
str(master.uuid))
_set_group_master_replication(group, None, update_only)
def _setup_sync(shard_id, source_group_id, destn_group_id, split_value,
prune_limit, cmd):
"""sync the source and the destination groups.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation
"""
source_group = Group.fetch(source_group_id)
if source_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(source_group_id, ))
destination_group = Group.fetch(destn_group_id)
if destination_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(destn_group_id, ))
master = MySQLServer.fetch(source_group.master)
if master is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
master.connect()
slave = MySQLServer.fetch(destination_group.master)
if slave is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
slave.connect()
#Synchronize until the slave catches up with the master.
_replication.synchronize_with_read_only(slave, master)
#Reset replication once the syncing is done.
_replication.stop_slave(slave, wait=True)
_replication.reset_slave(slave, clean=True)
#Trigger changing the mappings for the shard that was copied
_events.trigger_within_procedure(
SETUP_RESHARDING_SWITCH,
shard_id,
source_group_id,
destn_group_id,
split_value,
prune_limit,
cmd
)
def _setup_sync(shard_id, source_group_id, destn_group_id, split_value,
prune_limit, cmd):
"""sync the source and the destination groups.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation
"""
source_group = Group.fetch(source_group_id)
if source_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(source_group_id, ))
destination_group = Group.fetch(destn_group_id)
if destination_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(destn_group_id, ))
master = MySQLServer.fetch(source_group.master)
if master is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
master.connect()
slave = MySQLServer.fetch(destination_group.master)
if slave is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
slave.connect()
#Synchronize until the slave catches up with the master.
_replication.synchronize_with_read_only(slave, master)
#Reset replication once the syncing is done.
_replication.stop_slave(slave, wait=True)
_replication.reset_slave(slave, clean=True)
#Trigger changing the mappings for the shard that was copied
_events.trigger_within_procedure(
SETUP_RESHARDING_SWITCH,
shard_id,
source_group_id,
destn_group_id,
split_value,
prune_limit,
cmd
)
def _backup_source_shard(shard_id, source_group_id, destn_group_id,
split_value, prune_limit, cmd, update_only):
"""Backup the source shard.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation (move, split)
:update_only: Only update the state store and skip provisioning.
"""
backup_user = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_user'
)
backup_passwd = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_password'
)
mysqldump_binary = _services_utils.read_config_value(
_config.global_config,
'sharding',
'mysqldump_program'
)
source_group = Group.fetch(source_group_id)
move_source_server = _services_utils.fetch_backup_server(source_group)
#Do the backup of the group hosting the source shard.
backup_image = _backup.MySQLDump.backup(
move_source_server,
backup_user, backup_passwd,
mysqldump_binary
)
#Change the master for the server that is master of the group which hosts
#the destination shard.
_events.trigger_within_procedure(
RESTORE_SHARD_BACKUP,
shard_id,
source_group_id,
destn_group_id,
backup_image.path,
split_value,
prune_limit,
cmd
)
def _backup_source_shard(shard_id, source_group_id, destn_group_id,
split_value, prune_limit, cmd, update_only):
"""Backup the source shard.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation (move, split)
:update_only: Only update the state store and skip provisioning.
"""
backup_user = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_user'
)
backup_passwd = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_password'
)
mysqldump_binary = _services_utils.read_config_value(
_config.global_config,
'sharding',
'mysqldump_program'
)
source_group = Group.fetch(source_group_id)
move_source_server = _services_utils.fetch_backup_server(source_group)
#Do the backup of the group hosting the source shard.
backup_image = _backup.MySQLDump.backup(
move_source_server,
backup_user, backup_passwd,
mysqldump_binary
)
#Change the master for the server that is master of the group which hosts
#the destination shard.
_events.trigger_within_procedure(
RESTORE_SHARD_BACKUP,
shard_id,
source_group_id,
destn_group_id,
backup_image.path,
split_value,
prune_limit,
cmd
)
def _trigger_actions(server, update_only):
"""Trigger a failover if the server is a master.
"""
if not update_only:
group = _server.Group.fetch(server.group_id)
if group.master == server.uuid:
_LOGGER.info("Master (%s) in group (%s) has "
"been lost.", server.uuid, group.group_id)
_events.trigger_within_procedure("FAIL_OVER", group.group_id)
_events.trigger_within_procedure("SERVER_LOST", server.group_id,
str(server.uuid))
def _trigger_actions(server, update_only):
"""Trigger a failover if the server is a master.
"""
if not update_only:
group = _server.Group.fetch(server.group_id)
if group.master == server.uuid:
_LOGGER.info("Master (%s) in group (%s) has "
"been lost.", server.uuid, group.group_id)
_events.trigger_within_procedure("FAIL_OVER", group.group_id)
_events.trigger_within_procedure(
"SERVER_LOST", server.group_id, str(server.uuid)
)
def _wait_slave_fail(group_id, slave_uuid):
"""Wait until a slave processes its backlog.
"""
slave = _server.MySQLServer.fetch(_uuid.UUID(slave_uuid))
slave.connect()
try:
_utils.process_slave_backlog(slave)
except _errors.DatabaseError as error:
_LOGGER.warning(
"Error trying to process transactions in the relay log "
"for candidate (%s): %s.", slave, error)
_events.trigger_within_procedure(CHANGE_TO_CANDIDATE, group_id, slave_uuid)
def _check_candidate_switch(group_id, slave_id):
"""Check if the candidate has all the features to become the new
master.
"""
allowed_status = (_server.MySQLServer.SECONDARY, _server.MySQLServer.SPARE)
group = _server.Group.fetch(group_id)
if not group.master:
raise _errors.GroupError(
"Group (%s) does not contain a valid "
"master. Please, run a promote or failover." % (group_id, )
)
slave = _retrieve_server(slave_id, group_id)
slave.connect()
if group.master == slave.uuid:
raise _errors.ServerError(
"Candidate slave (%s) is already master." % (slave_id, )
)
master_issues = _replication.check_master_issues(slave)
if master_issues:
raise _errors.ServerError(
"Server (%s) is not a valid candidate slave "
"due to the following reason(s): (%s)." %
(slave.uuid, master_issues)
)
slave_issues = _replication.check_slave_issues(slave)
if slave_issues:
raise _errors.ServerError(
"Server (%s) is not a valid candidate slave "
"due to the following reason: (%s)." %
(slave.uuid, slave_issues)
)
master_uuid = _replication.slave_has_master(slave)
if master_uuid is None or group.master != _uuid.UUID(master_uuid):
raise _errors.GroupError(
"The group's master (%s) is different from the candidate's "
"master (%s)." % (group.master, master_uuid)
)
if slave.status not in allowed_status:
raise _errors.ServerError("Server (%s) is faulty." % (slave_id, ))
_events.trigger_within_procedure(
BLOCK_WRITE_SWITCH, group_id, master_uuid, str(slave.uuid)
)
def _wait_slaves_switch(group_id, master_uuid, slave_uuid):
"""Synchronize candidate with master and also all the other slaves.
Note that this can be optimized as one may determine the set of
slaves that must be synchronized with the master.
"""
master = _server.MySQLServer.fetch(_uuid.UUID(master_uuid))
master.connect()
slave = _server.MySQLServer.fetch(_uuid.UUID(slave_uuid))
slave.connect()
_utils.synchronize(slave, master)
_do_wait_slaves_catch(group_id, master, [slave_uuid])
_events.trigger_within_procedure(CHANGE_TO_CANDIDATE, group_id, slave_uuid)
def _wait_slaves_switch(group_id, master_uuid, slave_uuid):
"""Synchronize candidate with master and also all the other slaves.
Note that this can be optimized as one may determine the set of
slaves that must be synchronized with the master.
"""
master = _server.MySQLServer.fetch(_uuid.UUID(master_uuid))
master.connect()
slave = _server.MySQLServer.fetch(_uuid.UUID(slave_uuid))
slave.connect()
_utils.synchronize(slave, master)
_do_wait_slaves_catch(group_id, master, [slave_uuid])
_events.trigger_within_procedure(CHANGE_TO_CANDIDATE, group_id, slave_uuid)
def _wait_slave_fail(group_id, slave_uuid):
"""Wait until a slave processes its backlog.
"""
slave = _server.MySQLServer.fetch(_uuid.UUID(slave_uuid))
slave.connect()
try:
_utils.process_slave_backlog(slave)
except _errors.DatabaseError as error:
_LOGGER.warning(
"Error (%s) trying to process transactions in the relay log "
"for candidate (%s).", slave, error
)
_events.trigger_within_procedure(CHANGE_TO_CANDIDATE, group_id, slave_uuid)
def _backup_server(source_uuid, host, port, mysqldump_binary,
mysqlclient_binary, config_file):
"""Backup the source server, given by the source_uuid.
:param source_uuid: The UUID of the source server.
:param host: The hostname of the destination server.
:param port: The port number of the destination server.
:param mysqldump_binary: The MySQL Dump Binary path.
:param mysqlclient_binary: The MySQL Client Binary path.
:param config_file: The complete path to the fabric configuration
file.
"""
source_server = _server.MySQLServer.fetch(source_uuid)
#Do the backup of the group hosting the source shard.
backup_image = _backup.MySQLDump.backup(
source_server,
config_file,
mysqldump_binary
)
_LOGGER.debug("Done with backup of server with uuid = %s.", source_uuid)
procedures = _events.trigger_within_procedure(
RESTORE_SERVER,
source_uuid,
host,
port,
backup_image.path,
mysqlclient_binary,
config_file
)
def _add_server(group_id, address, timeout, update_only):
"""Only add a server with a configuring status into a group.
The next step is responsible for setting up the server though.
For example, replication will be configured in the next step.
"""
group = _retrieve_group(group_id)
uuid = _lookup_uuid(address, timeout)
_check_server_exists(uuid)
server = _server.MySQLServer(uuid=_uuid.UUID(uuid), address=address)
# Add server to the state store.
_server.MySQLServer.add(server)
# Add server as a member in the group.
server.group_id = group_id
_events.trigger_within_procedure(CONFIGURE_NEW_SERVER, group_id,
str(server.uuid), update_only)
def _restore_shard_backup(shard_id, source_group_id, destn_group_id,
mysqlclient_binary, backup_image,
split_value, config_file, cmd):
"""Restore the backup on the destination Group.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param mysqlclient_binary: The fully qualified mysqlclient binary.
:param backup_image: The destination file that contains the backup
of the source shard.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param config_file: The complete path to the fabric configuration file.
:param cmd: Indicates the type of re-sharding operation
"""
destn_group = Group.fetch(destn_group_id)
if destn_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(destn_group_id, ))
#Build a backup image that will be used for restoring
bk_img = _backup.BackupImage(backup_image)
for destn_group_server in destn_group.servers():
destn_group_server.connect()
_backup.MySQLDump.restore_fabric_server(
destn_group_server,
bk_img,
config_file,
mysqlclient_binary
)
#Setup sync between the source and the destination groups.
_events.trigger_within_procedure(
SETUP_MOVE_SYNC,
shard_id,
source_group_id,
destn_group_id,
split_value,
cmd
)
def _backup_source_shard(shard_id, source_group_id, destn_group_id,
mysqldump_binary, mysqlclient_binary, split_value,
config_file, prune_limit, cmd, update_only):
"""Backup the source shard.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param mysqldump_binary: The fully qualified mysqldump binary.
:param mysqlclient_binary: The fully qualified mysql client binary.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param config_file: The complete path to the fabric configuration file.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation (move, split)
:update_only: Only update the state store and skip provisioning.
"""
source_group = Group.fetch(source_group_id)
move_source_server = _services_utils.fetch_backup_server(source_group)
#Do the backup of the group hosting the source shard.
backup_image = _backup.MySQLDump.backup(
move_source_server,
config_file,
mysqldump_binary
)
#Change the master for the server that is master of the group which hosts
#the destination shard.
_events.trigger_within_procedure(
RESTORE_SHARD_BACKUP,
shard_id,
source_group_id,
destn_group_id,
mysqlclient_binary,
backup_image.path,
split_value,
config_file,
prune_limit,
cmd
)
def _add_server(group_id, address, timeout, update_only):
"""Only add a server with a configuring status into a group.
The next step is responsible for setting up the server though.
For example, replication will be configured in the next step.
"""
group = _retrieve_group(group_id)
uuid = _lookup_uuid(address, timeout)
_check_server_exists(uuid)
server = _server.MySQLServer(uuid=_uuid.UUID(uuid), address=address)
# Add server to the state store.
_server.MySQLServer.add(server)
# Add server as a member in the group.
server.group_id = group_id
_events.trigger_within_procedure(
CONFIGURE_NEW_SERVER, group_id, str(server.uuid), update_only
)
def _set_status_faulty(server, update_only):
"""Set server's status to fauly and trigger a failover if the server
is a master.
This function assumes that the SERVER_LOST event is executed before
the FAIL_OVER event.
"""
server.status = _server.MySQLServer.FAULTY
_events.trigger_within_procedure("SERVER_LOST", server.group_id,
str(server.uuid))
if not update_only:
_server.ConnectionPool().purge_connections(server.uuid)
group = _server.Group.fetch(server.group_id)
if group.master == server.uuid:
_LOGGER.info("Master (%s) in group (%s) has "
"been lost.", server.uuid, group.group_id)
_events.trigger_within_procedure("FAIL_OVER", group.group_id)
def _check_candidate_switch(group_id, slave_id):
"""Check if the candidate has all the features to become the new
master.
"""
allowed_status = (_server.MySQLServer.SECONDARY, _server.MySQLServer.SPARE)
group = _server.Group.fetch(group_id)
if not group.master:
raise _errors.GroupError("Group (%s) does not contain a valid "
"master. Please, run a promote or failover." %
(group_id, ))
slave = _retrieve_server(slave_id, group_id)
slave.connect()
if group.master == slave.uuid:
raise _errors.ServerError("Candidate slave (%s) is already master." %
(slave_id, ))
master_issues, why_master_issues = _replication.check_master_issues(slave)
if master_issues:
raise _errors.ServerError("Server (%s) is not a valid candidate slave "
"due to the following reason(s): (%s)." %
(slave.uuid, why_master_issues))
slave_issues, why_slave_issues = _replication.check_slave_issues(slave)
if slave_issues:
raise _errors.ServerError("Server (%s) is not a valid candidate slave "
"due to the following reason: (%s)." %
(slave.uuid, why_slave_issues))
master_uuid = _replication.slave_has_master(slave)
if master_uuid is None or group.master != _uuid.UUID(master_uuid):
raise _errors.GroupError(
"The group's master (%s) is different from the candidate's "
"master (%s)." % (group.master, master_uuid))
if slave.status not in allowed_status:
raise _errors.ServerError("Server (%s) is faulty." % (slave_id, ))
_events.trigger_within_procedure(BLOCK_WRITE_SWITCH, group_id, master_uuid,
str(slave.uuid))
def _set_status_faulty(server, update_only):
"""Set server's status to fauly and trigger a failover if the server
is a master.
This function assumes that the SERVER_LOST event is executed before
the FAIL_OVER event.
"""
server.status = _server.MySQLServer.FAULTY
_server.ConnectionManager().kill_connections(server)
if not update_only:
group = _server.Group.fetch(server.group_id)
if group.master == server.uuid:
_LOGGER.info("Master (%s) in group (%s) has "
"been lost.", server.uuid, group.group_id)
_events.trigger_within_procedure("FAIL_OVER", group.group_id)
_events.trigger_within_procedure(
"SERVER_LOST", server.group_id, str(server.uuid)
)
def _define_ha_operation(group_id, slave_id, update_only):
"""Define which operation must be called based on the master's status
and whether the candidate slave is provided or not.
"""
fail_over = True
group = _server.Group.fetch(group_id)
if not group:
raise _errors.GroupError("Group (%s) does not exist." % (group_id, ))
if update_only and not slave_id:
raise _errors.ServerError(
"The new master must be specified through --slave-uuid if "
"--update-only is set."
)
### Deactivate FailureDetector during failover/switchover.
if _detector.FailureDetector.is_active(group_id):
### Deactivate FailureDetector during failover/switchover.
_detector.FailureDetector.was_active = True
group.status = _server.Group.INACTIVE
_detector.FailureDetector.unregister_group(group_id)
if group.master:
master = _server.MySQLServer.fetch(group.master)
if master.status != _server.MySQLServer.FAULTY:
if update_only:
_do_block_write_master(group_id, str(group.master), update_only)
fail_over = False
if update_only:
# Check whether the server is registered or not.
_retrieve_server(slave_id, group_id)
_change_to_candidate(group_id, slave_id, update_only)
return
if fail_over:
if not slave_id:
_events.trigger_within_procedure(FIND_CANDIDATE_FAIL, group_id)
else:
_events.trigger_within_procedure(CHECK_CANDIDATE_FAIL, group_id,
slave_id
)
else:
if not slave_id:
_events.trigger_within_procedure(FIND_CANDIDATE_SWITCH, group_id)
else:
_events.trigger_within_procedure(CHECK_CANDIDATE_SWITCH, group_id,
slave_id
)
def _backup_server(source_uuid, host, port):
"""Backup the source server, given by the source_uuid.
:param source_uuid: The UUID of the source server.
:param host: The hostname of the destination server.
:param port: The port number of the destination server.
"""
backup_user = _services_utils.read_config_value(_config.global_config,
'servers', 'backup_user')
backup_passwd = _services_utils.read_config_value(_config.global_config,
'servers',
'backup_password')
mysqldump_binary = _services_utils.read_config_value(
_config.global_config, 'sharding', 'mysqldump_program')
source_server = _server.MySQLServer.fetch(source_uuid)
#Do the backup of the group hosting the source shard.
backup_image = _backup.MySQLDump.backup(source_server, backup_user,
backup_passwd, mysqldump_binary)
_LOGGER.debug("Done with backup of server with uuid = %s.", source_uuid)
_events.trigger_within_procedure(RESTORE_SERVER, source_uuid, host, port,
backup_image.path)
def _backup_server(source_uuid, host, port):
"""Backup the source server, given by the source_uuid.
:param source_uuid: The UUID of the source server.
:param host: The hostname of the destination server.
:param port: The port number of the destination server.
"""
backup_user = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_user'
)
backup_passwd = _services_utils.read_config_value(
_config.global_config,
'servers',
'backup_password'
)
mysqldump_binary = _services_utils.read_config_value(
_config.global_config,
'sharding',
'mysqldump_program'
)
source_server = _server.MySQLServer.fetch(source_uuid)
#Do the backup of the group hosting the source shard.
backup_image = _backup.MySQLDump.backup(
source_server,
backup_user, backup_passwd,
mysqldump_binary
)
_LOGGER.debug("Done with backup of server with uuid = %s.", source_uuid)
_events.trigger_within_procedure(
RESTORE_SERVER,
source_uuid,
host,
port,
backup_image.path
)
def _set_server_status_spare(server, update_only):
"""Set server's status to spare. If the server has faulty or
configuring status, it might happen that it is not properly
configured. So the server's status is temporarily set to
configuring and the proper configuration actions are taken
in the next step.
"""
allowed_status = [
_server.MySQLServer.SECONDARY, _server.MySQLServer.FAULTY,
_server.MySQLServer.CONFIGURING
]
status = _server.MySQLServer.SPARE
mode = _server.MySQLServer.OFFLINE
previous_status = server.status
if previous_status in \
(_server.MySQLServer.FAULTY, _server.MySQLServer.CONFIGURING):
if server.is_alive():
_events.trigger_within_procedure(CONFIGURE_FAULTY_SERVER,
str(server.uuid), previous_status,
update_only)
status = _server.MySQLServer.CONFIGURING
_do_set_status(server, allowed_status, status, mode, update_only)
def _define_ha_operation(group_id, slave_id, update_only):
"""Define which operation must be called based on the master's status
and whether the candidate slave is provided or not.
"""
fail_over = True
group = _server.Group.fetch(group_id)
if not group:
raise _errors.GroupError("Group (%s) does not exist." % (group_id, ))
if update_only and not slave_id:
raise _errors.ServerError(
"The new master must be specified through --slave-uuid if "
"--update-only is set.")
if group.master:
master = _server.MySQLServer.fetch(group.master)
if master.status != _server.MySQLServer.FAULTY:
if update_only:
_do_block_write_master(group_id, str(group.master),
update_only)
fail_over = False
if update_only:
# Check whether the server is registered or not.
_retrieve_server(slave_id, group_id)
_change_to_candidate(group_id, slave_id, update_only)
return
if fail_over:
if not slave_id:
_events.trigger_within_procedure(FIND_CANDIDATE_FAIL, group_id)
else:
_events.trigger_within_procedure(CHECK_CANDIDATE_FAIL, group_id,
slave_id)
else:
if not slave_id:
_events.trigger_within_procedure(FIND_CANDIDATE_SWITCH, group_id)
else:
_events.trigger_within_procedure(CHECK_CANDIDATE_SWITCH, group_id,
slave_id)
def _backup_server(source_uuid, host, port, mysqldump_binary,
mysqlclient_binary, config_file):
"""Backup the source server, given by the source_uuid.
:param source_uuid: The UUID of the source server.
:param host: The hostname of the destination server.
:param port: The port number of the destination server.
:param mysqldump_binary: The MySQL Dump Binary path.
:param mysqlclient_binary: The MySQL Client Binary path.
:param config_file: The complete path to the fabric configuration
file.
"""
source_server = _server.MySQLServer.fetch(source_uuid)
#Do the backup of the group hosting the source shard.
backup_image = _backup.MySQLDump.backup(source_server, config_file,
mysqldump_binary)
_LOGGER.debug("Done with backup of server with uuid = %s.", source_uuid)
procedures = _events.trigger_within_procedure(RESTORE_SERVER, source_uuid,
host, port,
backup_image.path,
mysqlclient_binary,
config_file)
def _check_shard_information(shard_id, destn_group_id, mysqldump_binary,
mysqlclient_binary, split_value, config_file,
prune_limit, cmd, update_only):
"""Verify the sharding information before starting a re-sharding operation.
:param shard_id: The destination shard ID.
:param destn_group_id: The Destination group ID.
:param mysqldump_binary: The path to the mysqldump binary.
:param mysqlclient_binary: The path to the mysqlclient binary.
:param split_value: The point at which the sharding definition
should be split.
:param config_file: The complete path to the fabric configuration
file.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates if it is a split or a move being executed.
:param update_only: If the operation is a update only operation.
"""
if not _services_utils.is_valid_binary(mysqldump_binary):
raise _errors.ShardingError(_services_sharding.MYSQLDUMP_NOT_FOUND %
mysqldump_binary)
if not _services_utils.is_valid_binary(mysqlclient_binary):
raise _errors.ShardingError(_services_sharding.MYSQLCLIENT_NOT_FOUND %
mysqlclient_binary)
if cmd == "SPLIT":
range_sharding_spec, _, shard_mappings, _ = \
_services_sharding.verify_and_fetch_shard(shard_id)
upper_bound = \
SHARDING_SPECIFICATION_HANDLER[shard_mappings[0].type_name].\
get_upper_bound(
range_sharding_spec.lower_bound,
range_sharding_spec.shard_mapping_id,
shard_mappings[0].type_name
)
#If the underlying sharding scheme is a HASH. When a shard is split,
#all the tables that are part of the shard, have the same sharding
#scheme. All the shard mappings associated with this shard_id will be
#of the same sharding type. Hence it is safe to use one of the shard
#mappings.
if shard_mappings[0].type_name == "HASH":
if split_value is not None:
raise _errors.ShardingError(
_services_sharding.NO_LOWER_BOUND_FOR_HASH_SHARDING)
if upper_bound is None:
#While splitting a range, retrieve the next upper bound and
#find the mid-point, in the case where the next upper_bound
#is unavailable pick the maximum value in the set of values in
#the shard.
upper_bound = HashShardingSpecification.fetch_max_key(shard_id)
#Calculate the split value.
split_value = \
SHARDING_DATATYPE_HANDLER[shard_mappings[0].type_name].\
split_value(
range_sharding_spec.lower_bound,
upper_bound
)
elif split_value is not None:
if not (SHARDING_DATATYPE_HANDLER[shard_mappings[0].type_name].\
is_valid_split_value(
split_value, range_sharding_spec.lower_bound,
upper_bound
)
):
raise _errors.ShardingError(
_services_sharding.INVALID_LOWER_BOUND_VALUE %
(split_value, ))
elif split_value is None:
raise _errors.ShardingError(
_services_sharding.SPLIT_VALUE_NOT_DEFINED)
#Ensure that the group does not already contain a shard.
if Shards.lookup_shard_id(destn_group_id) is not None:
raise _errors.ShardingError(
_services_sharding.SHARD_MOVE_DESTINATION_NOT_EMPTY %
(destn_group_id, ))
#Fetch the group information for the source shard that
#needs to be moved.
source_shard = Shards.fetch(shard_id)
if source_shard is None:
raise _errors.ShardingError(_services_sharding.SHARD_NOT_FOUND %
(shard_id, ))
#Fetch the group_id and the group that hosts the source shard.
source_group_id = source_shard.group_id
destn_group = Group.fetch(destn_group_id)
if destn_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(destn_group_id, ))
if not update_only:
_events.trigger_within_procedure(BACKUP_SOURCE_SHARD, shard_id,
source_group_id, destn_group_id,
mysqldump_binary, mysqlclient_binary,
split_value, config_file, prune_limit,
cmd, update_only)
else:
_events.trigger_within_procedure(SETUP_RESHARDING_SWITCH, shard_id,
source_group_id, destn_group_id,
split_value, prune_limit, cmd,
update_only)
def _add_shard(shard_mapping_id, groupid_lb_list, state, update_only=False):
"""Add the RANGE shard specification. This represents a single instance
of a shard specification that maps a key RANGE to a server.
:param shard_mapping_id: The unique identification for a shard mapping.
:param groupid_lb_list: The list of group_id, lower_bounds pairs in the
format, group_id/lower_bound, group_id/lower_bound... .
:param state: Indicates whether a given shard is ENABLED or DISABLED
:param update_only: Only update the state store and skip adding range checks.
:return: True if the add succeeded.
False otherwise.
:raises: ShardingError If the group on which the shard is being
created does not exist,
If the shard_mapping_id is not found,
If adding the shard definition fails,
If the state of the shard is an invalid
value,
If the range definition is invalid.
"""
shard_mapping = ShardMapping.fetch_shard_mapping_defn(shard_mapping_id)
if shard_mapping is None:
raise _errors.ShardingError(SHARD_MAPPING_NOT_FOUND % \
(shard_mapping_id, ))
schema_type = shard_mapping[1]
if len(RangeShardingSpecification.list(shard_mapping_id)) != 0:
raise _errors.ShardingError(SHARDS_ALREADY_EXIST)
group_id_list, lower_bound_list = \
_utils.get_group_lower_bound_list(groupid_lb_list)
if (len(group_id_list) != len(lower_bound_list)) and\
schema_type == "RANGE":
raise _errors.ShardingError(LOWER_BOUND_GROUP_ID_COUNT_MISMATCH)
if len(lower_bound_list) != 0 and schema_type == "HASH":
raise _errors.ShardingError(LOWER_BOUND_AUTO_GENERATED)
if schema_type in Shards.VALID_RANGE_SHARDING_TYPES:
for lower_bound in lower_bound_list:
if not SHARDING_DATATYPE_HANDLER[schema_type].\
is_valid_lower_bound(lower_bound):
raise _errors.ShardingError(
INVALID_LOWER_BOUND_VALUE % (lower_bound, ))
state = state.upper()
if state not in Shards.VALID_SHARD_STATES:
raise _errors.ShardingError(INVALID_SHARD_STATE % (state, ))
for index, group_id in enumerate(group_id_list):
shard = Shards.add(group_id, state)
shard_id = shard.shard_id
if schema_type == "HASH":
HashShardingSpecification.add(
shard_mapping_id,
shard_id
)
_LOGGER.debug(
"Added Shard (map id = %s, id = %s).",
shard_mapping_id,
shard_id
)
else:
range_sharding_specification = \
SHARDING_SPECIFICATION_HANDLER[schema_type].add(
shard_mapping_id,
lower_bound_list[index],
shard_id
)
_LOGGER.debug(
"Added Shard (map id = %s, lower bound = %s, id = %s).",
range_sharding_specification.shard_mapping_id,
range_sharding_specification.lower_bound,
range_sharding_specification.shard_id
)
if not update_only:
#If the shard is added in a DISABLED state do not setup replication
#with the primary of the global group. Basically setup replication only
#if the shard is ENABLED.
if state == "ENABLED":
_setup_shard_group_replication(shard_id)
if not update_only:
#Add the shard limits into the metadata present in each of the shards.
_events.trigger_within_procedure(
ADD_SHARD_RANGE_CHECK,
shard_mapping_id,
schema_type
)
def _setup_move_sync(shard_id, source_group_id, destn_group_id, split_value,
cmd):
"""Setup replication between the source and the destination groups and
ensure that they are in sync.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param cmd: Indicates the type of re-sharding operation
"""
source_group = Group.fetch(source_group_id)
if source_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(source_group_id, ))
destination_group = Group.fetch(destn_group_id)
if destination_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(destination_group_id, ))
master = MySQLServer.fetch(source_group.master)
if master is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
master.connect()
slave = MySQLServer.fetch(destination_group.master)
if slave is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
slave.connect()
#Stop and reset any slave that might be running on the slave server.
_replication.stop_slave(slave, wait=True)
_replication.reset_slave(slave, clean=True)
#Change the master to the shard group master.
_replication.switch_master(slave, master, master. user, master.passwd)
#Start the slave so that syncing of the data begins
_replication.start_slave(slave, wait=True)
#Synchronize until the slave catches up with the master.
_replication.synchronize_with_read_only(slave, master)
#Reset replication once the syncing is done.
_replication.stop_slave(slave, wait=True)
_replication.reset_slave(slave, clean=True)
#Trigger changing the mappings for the shard that was copied
_events.trigger_within_procedure(
SETUP_RESHARDING_SWITCH,
shard_id,
source_group_id,
destn_group_id,
split_value,
cmd
)
def _find_candidate_fail(group_id):
"""Find the best candidate to replace the failed master.
"""
slave_uuid = _do_find_candidate(group_id, FIND_CANDIDATE_FAIL)
_events.trigger_within_procedure(CHECK_CANDIDATE_FAIL, group_id,
slave_uuid)
def check_properties_6(param_01, param_02):
"""Check properties 6.
"""
_events.trigger_within_procedure(
EVENT_CHECK_PROPERTIES_2, "NEW 01", "NEW 02"
)
def _block_write_switch(group_id, master_uuid, slave_uuid):
"""Block and disable write access to the current master.
"""
_do_block_write_master(group_id, master_uuid)
_events.trigger_within_procedure(WAIT_SLAVES_SWITCH, group_id, master_uuid,
slave_uuid)
def _find_candidate_fail(group_id):
"""Find the best candidate to replace the failed master.
"""
slave_uuid = _do_find_candidate(group_id, FIND_CANDIDATE_FAIL)
_events.trigger_within_procedure(CHECK_CANDIDATE_FAIL, group_id,
slave_uuid)
def _find_candidate_switch(group_id):
"""Find the best slave to replace the current master.
"""
slave_uuid = _do_find_candidate(group_id, FIND_CANDIDATE_SWITCH)
_events.trigger_within_procedure(CHECK_CANDIDATE_SWITCH, group_id,
slave_uuid)
def _find_candidate_switch(group_id):
"""Find the best slave to replace the current master.
"""
slave_uuid = _do_find_candidate(group_id, FIND_CANDIDATE_SWITCH)
_events.trigger_within_procedure(CHECK_CANDIDATE_SWITCH, group_id,
slave_uuid)
def _setup_shard_switch_split(shard_id, source_group_id, destination_group_id,
split_value, prune_limit, cmd, update_only):
"""Setup the moved shard to map to the new group.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation.
:update_only: Only update the state store and skip provisioning.
"""
#Fetch the Range sharding specification.
range_sharding_spec, source_shard, shard_mappings, shard_mapping_defn = \
_services_sharding.verify_and_fetch_shard(shard_id)
#Disable the old shard
source_shard.disable()
#Remove the old shard.
range_sharding_spec.remove()
source_shard.remove()
destination_group = Group.fetch(destination_group_id)
if destination_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(destination_group_id, ))
destn_group_master = MySQLServer.fetch(destination_group.master)
if destn_group_master is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
destn_group_master.connect()
#Make the destination group as read only to disable updates until the
#connectors update their caches, thus avoiding inconsistency.
destn_group_master.read_only = True
#Add the new shards. Generate new shard IDs for the shard being
#split and also for the shard that is created as a result of the split.
new_shard_1 = Shards.add(source_shard.group_id, "DISABLED")
new_shard_2 = Shards.add(destination_group_id, "DISABLED")
#Both of the shard mappings associated with this shard_id should
#be of the same sharding type. Hence it is safe to use one of the
#shard mappings.
if shard_mappings[0].type_name == "HASH":
#In the case of a split involving a HASH sharding scheme,
#the shard that is split gets a new shard_id, while the split
#gets the new computed lower_bound and also a new shard id.
#NOTE: How the shard that is split retains its lower_bound.
HashShardingSpecification.add_hash_split(
range_sharding_spec.shard_mapping_id,
new_shard_1.shard_id,
range_sharding_spec.lower_bound
)
HashShardingSpecification.add_hash_split(
range_sharding_spec.shard_mapping_id,
new_shard_2.shard_id,
split_value
)
else:
#Add the new ranges. Note that the shard being split retains
#its lower_bound, while the new shard gets the computed,
#lower_bound.
RangeShardingSpecification.add(
range_sharding_spec.shard_mapping_id,
range_sharding_spec.lower_bound,
new_shard_1.shard_id
)
RangeShardingSpecification.add(
range_sharding_spec.shard_mapping_id,
split_value,
new_shard_2.shard_id
)
#The sleep ensures that the connector have refreshed their caches with the
#new shards that have been added as a result of the split.
time.sleep(_utils.TTL)
#The source shard group master would have been marked as read only
#during the sync. Remove the read_only flag.
source_group = Group.fetch(source_group_id)
if source_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(source_group_id, ))
source_group_master = MySQLServer.fetch(source_group.master)
if source_group_master is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
source_group_master.connect()
#Kill all the existing connections on the servers
source_group.kill_connections_on_servers()
#Allow connections on the source group master
source_group_master.read_only = False
#Allow connections on the destination group master
destn_group_master.read_only = False
#Setup replication for the new group from the global server
_group_replication.setup_group_replication \
(shard_mapping_defn[2], destination_group_id)
#Enable the split shards
new_shard_1.enable()
new_shard_2.enable()
#Trigger changing the mappings for the shard that was copied
if not update_only:
_events.trigger_within_procedure(
PRUNE_SHARDS, new_shard_1.shard_id, new_shard_2.shard_id, prune_limit
)
def _add_shard(shard_mapping_id, groupid_lb_list, state, update_only=False):
"""Add the RANGE shard specification. This represents a single instance
of a shard specification that maps a key RANGE to a server.
:param shard_mapping_id: The unique identification for a shard mapping.
:param groupid_lb_list: The list of group_id, lower_bounds pairs in the
format, group_id/lower_bound, group_id/lower_bound... .
:param state: Indicates whether a given shard is ENABLED or DISABLED
:param update_only: Only update the state store and skip adding range checks.
:return: True if the add succeeded.
False otherwise.
:raises: ShardingError If the group on which the shard is being
created does not exist,
If the shard_mapping_id is not found,
If adding the shard definition fails,
If the state of the shard is an invalid
value,
If the range definition is invalid.
"""
shard_mapping = ShardMapping.fetch_shard_mapping_defn(shard_mapping_id)
if shard_mapping is None:
raise _errors.ShardingError(SHARD_MAPPING_NOT_FOUND % \
(shard_mapping_id, ))
schema_type = shard_mapping[1]
if len(RangeShardingSpecification.list(shard_mapping_id)) != 0:
raise _errors.ShardingError(SHARDS_ALREADY_EXIST)
group_id_list, lower_bound_list = \
_utils.get_group_lower_bound_list(groupid_lb_list)
if (len(group_id_list) != len(lower_bound_list)) and\
schema_type == "RANGE":
raise _errors.ShardingError(LOWER_BOUND_GROUP_ID_COUNT_MISMATCH)
if len(lower_bound_list) != 0 and schema_type == "HASH":
raise _errors.ShardingError(LOWER_BOUND_AUTO_GENERATED)
if schema_type in Shards.VALID_RANGE_SHARDING_TYPES:
for lower_bound in lower_bound_list:
if not SHARDING_DATATYPE_HANDLER[schema_type].\
is_valid_lower_bound(lower_bound):
raise _errors.ShardingError(INVALID_LOWER_BOUND_VALUE %
(lower_bound, ))
state = state.upper()
if state not in Shards.VALID_SHARD_STATES:
raise _errors.ShardingError(INVALID_SHARD_STATE % (state, ))
for index, group_id in enumerate(group_id_list):
shard = Shards.add(group_id, state)
shard_id = shard.shard_id
if schema_type == "HASH":
HashShardingSpecification.add(shard_mapping_id, shard_id)
_LOGGER.debug("Added Shard (map id = %s, id = %s).",
shard_mapping_id, shard_id)
else:
range_sharding_specification = \
SHARDING_SPECIFICATION_HANDLER[schema_type].add(
shard_mapping_id,
lower_bound_list[index],
shard_id
)
_LOGGER.debug(
"Added Shard (map id = %s, lower bound = %s, id = %s).",
range_sharding_specification.shard_mapping_id,
range_sharding_specification.lower_bound,
range_sharding_specification.shard_id)
if not update_only:
#If the shard is added in a DISABLED state do not setup replication
#with the primary of the global group. Basically setup replication only
#if the shard is ENABLED.
if state == "ENABLED":
_setup_shard_group_replication(shard_id)
if not update_only:
#Add the shard limits into the metadata present in each of the shards.
_events.trigger_within_procedure(ADD_SHARD_RANGE_CHECK,
shard_mapping_id, schema_type)
def _setup_shard_switch_split(shard_id, source_group_id, destination_group_id,
split_value, prune_limit, cmd, update_only):
"""Setup the moved shard to map to the new group.
:param shard_id: The shard ID of the shard that needs to be moved.
:param source_group_id: The group_id of the source shard.
:param destn_group_id: The ID of the group to which the shard needs to
be moved.
:param split_value: Indicates the value at which the range for the
particular shard will be split. Will be set only
for shard split operations.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates the type of re-sharding operation.
:update_only: Only update the state store and skip provisioning.
"""
#Fetch the Range sharding specification.
range_sharding_spec, source_shard, shard_mappings, shard_mapping_defn = \
_services_sharding.verify_and_fetch_shard(shard_id)
#Disable the old shard
source_shard.disable()
#Remove the old shard.
range_sharding_spec.remove()
source_shard.remove()
destination_group = Group.fetch(destination_group_id)
if destination_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(destination_group_id, ))
destn_group_master = MySQLServer.fetch(destination_group.master)
if destn_group_master is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
destn_group_master.connect()
#Make the destination group as read only to disable updates until the
#connectors update their caches, thus avoiding inconsistency.
destn_group_master.read_only = True
#Add the new shards. Generate new shard IDs for the shard being
#split and also for the shard that is created as a result of the split.
new_shard_1 = Shards.add(source_shard.group_id, "DISABLED")
new_shard_2 = Shards.add(destination_group_id, "DISABLED")
#Both of the shard mappings associated with this shard_id should
#be of the same sharding type. Hence it is safe to use one of the
#shard mappings.
if shard_mappings[0].type_name == "HASH":
#In the case of a split involving a HASH sharding scheme,
#the shard that is split gets a new shard_id, while the split
#gets the new computed lower_bound and also a new shard id.
#NOTE: How the shard that is split retains its lower_bound.
HashShardingSpecification.add_hash_split(
range_sharding_spec.shard_mapping_id, new_shard_1.shard_id,
range_sharding_spec.lower_bound)
HashShardingSpecification.add_hash_split(
range_sharding_spec.shard_mapping_id, new_shard_2.shard_id,
split_value)
else:
#Add the new ranges. Note that the shard being split retains
#its lower_bound, while the new shard gets the computed,
#lower_bound.
RangeShardingSpecification.add(range_sharding_spec.shard_mapping_id,
range_sharding_spec.lower_bound,
new_shard_1.shard_id)
RangeShardingSpecification.add(range_sharding_spec.shard_mapping_id,
split_value, new_shard_2.shard_id)
#The sleep ensures that the connector have refreshed their caches with the
#new shards that have been added as a result of the split.
time.sleep(_utils.TTL)
#The source shard group master would have been marked as read only
#during the sync. Remove the read_only flag.
source_group = Group.fetch(source_group_id)
if source_group is None:
raise _errors.ShardingError(_services_sharding.SHARD_GROUP_NOT_FOUND %
(source_group_id, ))
source_group_master = MySQLServer.fetch(source_group.master)
if source_group_master is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_MASTER_NOT_FOUND)
source_group_master.connect()
#Kill all the existing connections on the servers
source_group.kill_connections_on_servers()
#Allow connections on the source group master
source_group_master.read_only = False
#Allow connections on the destination group master
destn_group_master.read_only = False
#Setup replication for the new group from the global server
_group_replication.setup_group_replication \
(shard_mapping_defn[2], destination_group_id)
#Enable the split shards
new_shard_1.enable()
new_shard_2.enable()
#Trigger changing the mappings for the shard that was copied
if not update_only:
_events.trigger_within_procedure(PRUNE_SHARDS, new_shard_1.shard_id,
new_shard_2.shard_id, prune_limit)
def _check_shard_information(shard_id, destn_group_id, mysqldump_binary,
mysqlclient_binary, split_value, config_file, prune_limit, cmd,
update_only):
"""Verify the sharding information before starting a re-sharding operation.
:param shard_id: The destination shard ID.
:param destn_group_id: The Destination group ID.
:param mysqldump_binary: The path to the mysqldump binary.
:param mysqlclient_binary: The path to the mysqlclient binary.
:param split_value: The point at which the sharding definition
should be split.
:param config_file: The complete path to the fabric configuration
file.
:param prune_limit: The number of DELETEs that should be
done in one batch.
:param cmd: Indicates if it is a split or a move being executed.
:param update_only: If the operation is a update only operation.
"""
if not _services_utils.is_valid_binary(mysqldump_binary):
raise _errors.ShardingError(
_services_sharding.MYSQLDUMP_NOT_FOUND % mysqldump_binary)
if not _services_utils.is_valid_binary(mysqlclient_binary):
raise _errors.ShardingError(
_services_sharding.MYSQLCLIENT_NOT_FOUND % mysqlclient_binary)
if cmd == "SPLIT":
range_sharding_spec, _, shard_mappings, _ = \
_services_sharding.verify_and_fetch_shard(shard_id)
upper_bound = \
SHARDING_SPECIFICATION_HANDLER[shard_mappings[0].type_name].\
get_upper_bound(
range_sharding_spec.lower_bound,
range_sharding_spec.shard_mapping_id,
shard_mappings[0].type_name
)
#If the underlying sharding scheme is a HASH. When a shard is split,
#all the tables that are part of the shard, have the same sharding
#scheme. All the shard mappings associated with this shard_id will be
#of the same sharding type. Hence it is safe to use one of the shard
#mappings.
if shard_mappings[0].type_name == "HASH":
if split_value is not None:
raise _errors.ShardingError(
_services_sharding.NO_LOWER_BOUND_FOR_HASH_SHARDING
)
if upper_bound is None:
#While splitting a range, retrieve the next upper bound and
#find the mid-point, in the case where the next upper_bound
#is unavailable pick the maximum value in the set of values in
#the shard.
upper_bound = HashShardingSpecification.fetch_max_key(shard_id)
#Calculate the split value.
split_value = \
SHARDING_DATATYPE_HANDLER[shard_mappings[0].type_name].\
split_value(
range_sharding_spec.lower_bound,
upper_bound
)
elif split_value is not None:
if not (SHARDING_DATATYPE_HANDLER[shard_mappings[0].type_name].\
is_valid_split_value(
split_value, range_sharding_spec.lower_bound,
upper_bound
)
):
raise _errors.ShardingError(
_services_sharding.INVALID_LOWER_BOUND_VALUE %
(split_value, )
)
elif split_value is None:
raise _errors.ShardingError(
_services_sharding.SPLIT_VALUE_NOT_DEFINED
)
#Ensure that the group does not already contain a shard.
if Shards.lookup_shard_id(destn_group_id) is not None:
raise _errors.ShardingError(
_services_sharding.SHARD_MOVE_DESTINATION_NOT_EMPTY %
(destn_group_id, )
)
#Fetch the group information for the source shard that
#needs to be moved.
source_shard = Shards.fetch(shard_id)
if source_shard is None:
raise _errors.ShardingError(
_services_sharding.SHARD_NOT_FOUND % (shard_id, ))
#Fetch the group_id and the group that hosts the source shard.
source_group_id = source_shard.group_id
destn_group = Group.fetch(destn_group_id)
if destn_group is None:
raise _errors.ShardingError(
_services_sharding.SHARD_GROUP_NOT_FOUND %
(destn_group_id, ))
if not update_only:
_events.trigger_within_procedure(
BACKUP_SOURCE_SHARD, shard_id, source_group_id, destn_group_id,
mysqldump_binary, mysqlclient_binary, split_value, config_file,
prune_limit, cmd, update_only
)
else:
_events.trigger_within_procedure(
SETUP_RESHARDING_SWITCH, shard_id, source_group_id, destn_group_id,
split_value, prune_limit, cmd, update_only
)