def _report_failure(self, server):
"""Mark the server as faulty and report a failure.
The thread is created to allow the built-in failure detector
to continue monitoring the servers so that if the report failure
hangs, it will kill all connections to faulty servers thus
eventually freeing the thread.
Not though that the report failure is not crash-safe so it might
fail without promoting a new server to master. In the future, we
will circumvent this limitation.
"""
try:
_persistence.init_thread()
server.status = MySQLServer.FAULTY
self.__connection_manager.purge_connections(server)
procedures = trigger(
"REPORT_FAILURE", None, str(server.uuid),
threading.current_thread().name, MySQLServer.FAULTY, False
)
executor = _executor.Executor()
for procedure in procedures:
executor.wait_for_procedure(procedure)
_persistence.deinit_thread()
finally:
self.__thread_report_failure = False
def run(self):
"""Process registered requests.
"""
_LOGGER.info("Started XML-RPC-Session.")
try:
_persistence.init_thread()
except Exception as error:
_LOGGER.warning("Error connecting to backing store: (%s).", error)
SessionThread.local_thread.thread = self
while True:
request, client_address = self.__server.dequeue_request()
_LOGGER.debug(
"Processing request (%s) from (%s) through thread (%s).",
request, client_address, self
)
# There is no need to catch exceptions here because the method
# process_request_thread already does so. It is the main entry
# point in the code which means that any uncaught exception
# in the code will be reported as xmlrpclib.Fault.
self.__server.process_request_thread(request, client_address)
_LOGGER.debug(
"Finishing request (%s) from (%s) through thread (%s).",
request, client_address, self
)
if self.__is_shutdown:
self.__server.shutdown_now()
try:
_persistence.deinit_thread()
except Exception as error:
_LOGGER.warning("Error connecting to backing store: (%s).",
error)
def _run(self):
"""Function that verifies servers' availabilities.
"""
ignored_status = [MySQLServer.FAULTY]
quarantine = {}
interval = FailureDetector._DETECTION_INTERVAL
detections = FailureDetector._DETECTIONS
detection_timeout = FailureDetector._DETECTION_TIMEOUT
_persistence.init_thread()
while self.__check:
try:
unreachable = set()
group = Group.fetch(self.__group_id)
if group is not None:
for server in group.servers():
if server.status in ignored_status or \
MySQLServer.is_alive(server, detection_timeout):
if server.status == MySQLServer.FAULTY:
self.__connection_manager.purge_connections(
server
)
continue
unreachable.add(server.uuid)
_LOGGER.warning(
"Server (%s) in group (%s) is unreachable.",
server.uuid, self.__group_id
)
unstable = False
failed_attempts = 0
if server.uuid not in quarantine:
quarantine[server.uuid] = failed_attempts = 1
else:
failed_attempts = quarantine[server.uuid] + 1
quarantine[server.uuid] = failed_attempts
if failed_attempts >= detections:
unstable = True
can_set_faulty = group.can_set_server_faulty(
server, get_time()
)
if unstable and can_set_faulty:
self._spawn_report_failure(server)
for uuid in quarantine.keys():
if uuid not in unreachable:
del quarantine[uuid]
except (_errors.ExecutorError, _errors.DatabaseError):
pass
except Exception as error:
_LOGGER.exception(error)
time.sleep(interval / detections)
_persistence.deinit_thread()
def run(self):
"""Process registered requests.
"""
_LOGGER.info("Started XML-RPC-Session.")
try:
_persistence.init_thread()
except Exception as error:
_LOGGER.warning("Error connecting to backing store: (%s).", error)
SessionThread.local_thread.thread = self
while True:
request, client_address = self.__server.dequeue_request()
_LOGGER.debug(
"Processing request (%s) from (%s) through thread (%s).",
request, client_address, self)
# There is no need to catch exceptions here because the method
# process_request_thread already does so. It is the main entry
# point in the code which means that any uncaught exception
# in the code will be reported as xmlrpclib.Fault.
self.__server.process_request_thread(request, client_address)
_LOGGER.debug(
"Finishing request (%s) from (%s) through thread (%s).",
request, client_address, self)
if self.__is_shutdown:
self.__server.shutdown_now()
try:
_persistence.deinit_thread()
except Exception as error:
_LOGGER.warning("Error connecting to backing store: (%s).", error)
def _run(self):
"""Function that verifies servers' availabilities.
"""
from mysql.fabric.server import (
Group,
MySQLServer,
ConnectionManager,
)
ignored_status = [MySQLServer.FAULTY]
quarantine = {}
interval = FailureDetector._DETECTION_INTERVAL
detections = FailureDetector._DETECTIONS
detection_timeout = FailureDetector._DETECTION_TIMEOUT
connection_manager = ConnectionManager()
slave_deep_checks = FailureDetector._SLAVE_DEEP_CHECKS
_persistence.init_thread()
while self.__check:
try:
unreachable = set()
group = Group.fetch(self.__group_id)
if group is not None:
for server in group.servers():
if server.status in ignored_status:
### Server is FAULTY
connection_manager.kill_connections(server)
continue
else:
### Server is Not FAULTY
if MySQLServer.is_alive(server, detection_timeout):
### Server is alive
### check depends on `slave_deep_checks` parameter
if slave_deep_checks:
### When server is alive and status != FAULTY
is_master= (group.master == server.uuid)
if not is_master:
### Checking master is dead or alive.
master_server = MySQLServer.fetch(group.master)
if MySQLServer.is_alive(master_server, detection_timeout):
### Checking is replication valid or not if master is alive.
server.connect()
slave_issues, why_slave_issues = \
_replication.check_slave_issues(server)
if slave_issues:
if (why_slave_issues['io_error'] and \
why_slave_issues['io_errno'] == 2003):
### Nothing to do during reconnecting, just logging
_LOGGER.info(why_slave_issues)
else:
### If slave threads are not running, set status to SPARE
server.status = MySQLServer.SPARE
### Done slave_issues.
server.disconnect()
### Endif MySQLServer.is_alive(master_server, detection_timeout)
### Endif not is_master
### Endif slave_deep_checks
continue
### Else MySQLServer.is_alive(server, detection_timeout)
else:
unreachable.add(server.uuid)
_LOGGER.warning(
"Server (%s) in group (%s) is unreachable.",
server.uuid, self.__group_id
)
unstable = False
failed_attempts = 0
if server.uuid not in quarantine:
quarantine[server.uuid] = failed_attempts = 1
else:
failed_attempts = quarantine[server.uuid] + 1
quarantine[server.uuid] = failed_attempts
if failed_attempts >= detections:
unstable = True
can_set_faulty = group.can_set_server_faulty(
server, get_time()
)
if unstable and can_set_faulty:
# We have to make this transactional and make the
# failover (i.e. report failure) robust to failures.
# Otherwise, a master might be set to faulty and
# a new one never promoted.
server.status = MySQLServer.FAULTY
connection_manager.kill_connections(server)
procedures = trigger("REPORT_FAILURE", None,
str(server.uuid),
threading.current_thread().name,
MySQLServer.FAULTY, False
)
executor = _executor.Executor()
for procedure in procedures:
executor.wait_for_procedure(procedure)
### Endif MySQLServer.is_alive(server, detection_timeout)
### Endif server.status in ignored_status
### End for server in group.servers()
### Endif group is not None
for uuid in quarantine.keys():
if uuid not in unreachable:
del quarantine[uuid]
except (_errors.ExecutorError, _errors.DatabaseError):
pass
except Exception as error:
_LOGGER.exception(error)
time.sleep(interval)
_persistence.deinit_thread()
def teardown_xmlrpc(proxy):
from mysql.fabric import persistence
proxy.manage.stop()
persistence.deinit_thread()
persistence.teardown()
def _run(self):
"""Function that verifies servers' availabilities.
"""
from mysql.fabric.server import (
Group,
MySQLServer,
ConnectionManager,
)
ignored_status = [MySQLServer.FAULTY]
quarantine = {}
interval = FailureDetector._DETECTION_INTERVAL
detections = FailureDetector._DETECTIONS
detection_timeout = FailureDetector._DETECTION_TIMEOUT
connection_manager = ConnectionManager()
_persistence.init_thread()
while self.__check:
try:
unreachable = set()
group = Group.fetch(self.__group_id)
if group is not None:
for server in group.servers():
if server.status in ignored_status or \
MySQLServer.is_alive(server, detection_timeout):
if server.status == MySQLServer.FAULTY:
connection_manager.kill_connections(server)
continue
unreachable.add(server.uuid)
_LOGGER.warning(
"Server (%s) in group (%s) is unreachable.",
server.uuid, self.__group_id
)
unstable = False
failed_attempts = 0
if server.uuid not in quarantine:
quarantine[server.uuid] = failed_attempts = 1
else:
failed_attempts = quarantine[server.uuid] + 1
quarantine[server.uuid] = failed_attempts
if failed_attempts >= detections:
unstable = True
can_set_faulty = group.can_set_server_faulty(
server, get_time()
)
if unstable and can_set_faulty:
# We have to make this transactional and make the
# failover (i.e. report failure) robust to failures.
# Otherwise, a master might be set to faulty and
# a new one never promoted.
server.status = MySQLServer.FAULTY
connection_manager.kill_connections(server)
procedures = trigger("REPORT_FAILURE", None,
str(server.uuid),
threading.current_thread().name,
MySQLServer.FAULTY, False
)
executor = _executor.Executor()
for procedure in procedures:
executor.wait_for_procedure(procedure)
for uuid in quarantine.keys():
if uuid not in unreachable:
del quarantine[uuid]
except (_errors.ExecutorError, _errors.DatabaseError):
pass
except Exception as error:
_LOGGER.exception(error)
time.sleep(interval / detections)
_persistence.deinit_thread()
def teardown_xmlrpc(proxy):
from mysql.fabric import persistence
proxy.manage.stop()
persistence.deinit_thread()
persistence.teardown()
