def _define_ha_operation(group_id, slave_id, update_only):
"""Define which operation must be called based on the master's status
and whether the candidate slave is provided or not.
"""
fail_over = True
group = _server.Group.fetch(group_id)
if not group:
raise _errors.GroupError("Group (%s) does not exist." % (group_id, ))
if update_only and not slave_id:
raise _errors.ServerError(
"The new master must be specified through --slave-uuid if "
"--update-only is set."
)
### Deactivate FailureDetector during failover/switchover.
if _detector.FailureDetector.is_active(group_id):
### Deactivate FailureDetector during failover/switchover.
_detector.FailureDetector.was_active = True
group.status = _server.Group.INACTIVE
_detector.FailureDetector.unregister_group(group_id)
if group.master:
master = _server.MySQLServer.fetch(group.master)
if master.status != _server.MySQLServer.FAULTY:
if update_only:
_do_block_write_master(group_id, str(group.master), update_only)
fail_over = False
if update_only:
# Check whether the server is registered or not.
_retrieve_server(slave_id, group_id)
_change_to_candidate(group_id, slave_id, update_only)
return
if fail_over:
if not slave_id:
_events.trigger_within_procedure(FIND_CANDIDATE_FAIL, group_id)
else:
_events.trigger_within_procedure(CHECK_CANDIDATE_FAIL, group_id,
slave_id
)
else:
if not slave_id:
_events.trigger_within_procedure(FIND_CANDIDATE_SWITCH, group_id)
else:
_events.trigger_within_procedure(CHECK_CANDIDATE_SWITCH, group_id,
slave_id
)
def _check_candidate_fail(group_id, slave_id):
"""Check if the candidate has all the prerequisites to become the new
master.
"""
allowed_status = (_server.MySQLServer.SECONDARY, _server.MySQLServer.SPARE)
group = _server.Group.fetch(group_id)
slave = _retrieve_server(slave_id, group_id)
slave.connect()
if group.master == slave.uuid:
raise _errors.ServerError("Candidate slave (%s) is already master." %
(slave_id, ))
master_issues, why_master_issues = _replication.check_master_issues(slave)
if master_issues:
raise _errors.ServerError("Server (%s) is not a valid candidate slave "
"due to the following reason(s): (%s)." %
(slave.uuid, why_master_issues))
if slave.status not in allowed_status:
raise _errors.ServerError("Server (%s) is faulty." % (slave_id, ))
_events.trigger_within_procedure(WAIT_SLAVE_FAIL, group_id,
str(slave.uuid))
def _check_candidate_fail(group_id, slave_id):
"""Check if the candidate has all the prerequisites to become the new
master.
"""
allowed_status = (_server.MySQLServer.SECONDARY, _server.MySQLServer.SPARE)
group = _server.Group.fetch(group_id)
slave = _retrieve_server(slave_id, group_id)
slave.connect()
if group.master == slave.uuid:
raise _errors.ServerError(
"Candidate slave (%s) is already master." % (slave_id, )
)
master_issues = _replication.check_master_issues(slave)
if master_issues:
raise _errors.ServerError(
"Server (%s) is not a valid candidate slave "
"due to the following reason(s): (%s)." %
(slave.uuid, master_issues)
)
if slave.status not in allowed_status:
raise _errors.ServerError("Server (%s) is faulty." % (slave_id, ))
_events.trigger_within_procedure(WAIT_SLAVE_FAIL, group_id, str(slave.uuid))
def _append_error_log(server_id, reporter, error):
"""Check whether the server exist and is not faulty and register
error log.
"""
server = _retrieve_server(server_id)
now = get_time()
_error_log.ErrorLog.add(server, now, reporter, error)
_LOGGER.warning("Reported issue (%s) for server (%s).", error, server.uuid)
return (now, server)
def _append_error_log(server_id, reporter, error):
"""Check whether the server exist and is not faulty and register
error log.
"""
server = _retrieve_server(server_id)
now = get_time()
_error_log.ErrorLog.add(server, now, reporter, error)
_LOGGER.warning("Reported issue (%s) for server (%s).", error, server.uuid)
return (now, server)
def _define_ha_operation(group_id, slave_id, update_only):
"""Define which operation must be called based on the master's status
and whether the candidate slave is provided or not.
"""
fail_over = True
group = _server.Group.fetch(group_id)
if not group:
raise _errors.GroupError("Group (%s) does not exist." % (group_id, ))
if update_only and not slave_id:
raise _errors.ServerError(
"The new master must be specified through --slave-uuid if "
"--update-only is set.")
if group.master:
master = _server.MySQLServer.fetch(group.master)
if master.status != _server.MySQLServer.FAULTY:
if update_only:
_do_block_write_master(group_id, str(group.master),
update_only)
fail_over = False
if update_only:
# Check whether the server is registered or not.
_retrieve_server(slave_id, group_id)
_change_to_candidate(group_id, slave_id, update_only)
return
if fail_over:
if not slave_id:
_events.trigger_within_procedure(FIND_CANDIDATE_FAIL, group_id)
else:
_events.trigger_within_procedure(CHECK_CANDIDATE_FAIL, group_id,
slave_id)
else:
if not slave_id:
_events.trigger_within_procedure(FIND_CANDIDATE_SWITCH, group_id)
else:
_events.trigger_within_procedure(CHECK_CANDIDATE_SWITCH, group_id,
slave_id)
def _check_candidate_switch(group_id, slave_id):
"""Check if the candidate has all the features to become the new
master.
"""
allowed_status = (_server.MySQLServer.SECONDARY, _server.MySQLServer.SPARE)
group = _server.Group.fetch(group_id)
if not group.master:
raise _errors.GroupError(
"Group (%s) does not contain a valid "
"master. Please, run a promote or failover." % (group_id, )
)
slave = _retrieve_server(slave_id, group_id)
slave.connect()
if group.master == slave.uuid:
raise _errors.ServerError(
"Candidate slave (%s) is already master." % (slave_id, )
)
master_issues = _replication.check_master_issues(slave)
if master_issues:
raise _errors.ServerError(
"Server (%s) is not a valid candidate slave "
"due to the following reason(s): (%s)." %
(slave.uuid, master_issues)
)
slave_issues = _replication.check_slave_issues(slave)
if slave_issues:
raise _errors.ServerError(
"Server (%s) is not a valid candidate slave "
"due to the following reason: (%s)." %
(slave.uuid, slave_issues)
)
master_uuid = _replication.slave_has_master(slave)
if master_uuid is None or group.master != _uuid.UUID(master_uuid):
raise _errors.GroupError(
"The group's master (%s) is different from the candidate's "
"master (%s)." % (group.master, master_uuid)
)
if slave.status not in allowed_status:
raise _errors.ServerError("Server (%s) is faulty." % (slave_id, ))
_events.trigger_within_procedure(
BLOCK_WRITE_SWITCH, group_id, master_uuid, str(slave.uuid)
)
def _append_error_log(server_id, reporter, error):
"""Check whether the server exist and is not faulty and register
error log.
"""
server = _retrieve_server(server_id)
if server.status == _server.MySQLServer.FAULTY:
raise _errors.ServerError("Server (%s) is already marked as faulty." %
(server.uuid, ))
_LOGGER.warning("Reported issue (%s) for server (%s).", error, server.uuid)
now = get_time()
_error_log.ErrorLog.add(server, now, reporter, error)
return (now, server)
def _append_error_log(server_id, reporter, error):
"""Check whether the server exist and is not faulty and register
error log.
"""
server = _retrieve_server(server_id)
if server.status == _server.MySQLServer.FAULTY:
raise _errors.ServerError(
"Server (%s) is already marked as faulty." % (server.uuid, )
)
_LOGGER.warning("Reported issue (%s) for server (%s).", error, server.uuid)
now = get_time()
_error_log.ErrorLog.add(server, now, reporter, error)
return (now, server)
def execute(self, server_id, reporter="unknown", error="unknown",
update_only=False, synchronous=True):
"""Report a server issue.
:param server_id: Servers's UUID or HOST:PORT.
:param reporter: Who has reported the issue, usually an IP address or a
host name.
:param error: Error that has been reported.
:param update_only: Only update the state store and skip provisioning.
"""
server = _retrieve_server(server_id)
_set_status_faulty(server)
procedures = _events.trigger(
REPORT_FAILURE, self.get_lockable_objects(), server_id, reporter,
error, update_only
)
return self.wait_for_procedures(procedures, synchronous)
def _check_candidate_switch(group_id, slave_id):
"""Check if the candidate has all the features to become the new
master.
"""
allowed_status = (_server.MySQLServer.SECONDARY, _server.MySQLServer.SPARE)
group = _server.Group.fetch(group_id)
if not group.master:
raise _errors.GroupError("Group (%s) does not contain a valid "
"master. Please, run a promote or failover." %
(group_id, ))
slave = _retrieve_server(slave_id, group_id)
slave.connect()
if group.master == slave.uuid:
raise _errors.ServerError("Candidate slave (%s) is already master." %
(slave_id, ))
master_issues, why_master_issues = _replication.check_master_issues(slave)
if master_issues:
raise _errors.ServerError("Server (%s) is not a valid candidate slave "
"due to the following reason(s): (%s)." %
(slave.uuid, why_master_issues))
slave_issues, why_slave_issues = _replication.check_slave_issues(slave)
if slave_issues:
raise _errors.ServerError("Server (%s) is not a valid candidate slave "
"due to the following reason: (%s)." %
(slave.uuid, why_slave_issues))
master_uuid = _replication.slave_has_master(slave)
if master_uuid is None or group.master != _uuid.UUID(master_uuid):
raise _errors.GroupError(
"The group's master (%s) is different from the candidate's "
"master (%s)." % (group.master, master_uuid))
if slave.status not in allowed_status:
raise _errors.ServerError("Server (%s) is faulty." % (slave_id, ))
_events.trigger_within_procedure(BLOCK_WRITE_SWITCH, group_id, master_uuid,
str(slave.uuid))
def execute(self,
server_id,
reporter="unknown",
error="unknown",
update_only=False,
synchronous=True):
"""Report a server issue.
:param server_id: Servers's UUID or HOST:PORT.
:param reporter: Who has reported the issue, usually an IP address or a
host name.
:param error: Error that has been reported.
:param update_only: Only update the state store and skip provisioning.
"""
server = _retrieve_server(server_id)
_set_status_faulty(server)
procedures = _events.trigger(REPORT_FAILURE,
self.get_lockable_objects(), server_id,
reporter, error, update_only)
return self.wait_for_procedures(procedures, synchronous)